Skip to content

Commit 3913a9f

Browse files
BYKBYK
authored
ci: Even better cache keys and granular caching (#3534)
Just starting up services for Snuba or Sentry migrations takes up to a minute sometimes and we do this even when there are no migrations, just because one of the Sentry or Snuba migrations change. This patch splits the caches up so only the necessary one runs, saving further time. It also uses the `LATEST_TAG` as the cache key for upgrade tests as the image versions or data will never change for a certain tag once it is release.
1 parent 63b6c0a commit 3913a9f

File tree

4 files changed

+59
-27
lines changed

4 files changed

+59
-27
lines changed

‎.github/workflows/test.yml

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -56,22 +56,13 @@ jobs:
5656
sudo curl -L https://github.com/docker/compose/releases/download/v2.26.0/docker-compose-`uname -s`-`uname -m` -o "/usr/local/lib/docker/cli-plugins/docker-compose"
5757
sudo chmod +x "/usr/local/lib/docker/cli-plugins/docker-compose"
5858
59-
- name: Compute Docker Volume Cache Key
60-
id: cache_key
61-
run: |
62-
source .env
63-
SENTRY_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SENTRY_IMAGE -c 'ls -Rv1rpq src/sentry/migrations/' | md5sum | cut -d ' ' -f 1)
64-
echo "SENTRY_MIGRATIONS_MD5=$SENTRY_MIGRATIONS_MD5" >> $GITHUB_OUTPUT
65-
SNUBA_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SNUBA_IMAGE -c 'ls -Rv1rpq snuba/snuba_migrations/**/*.py' | md5sum | cut -d ' ' -f 1)
66-
echo "SNUBA_MIGRATIONS_MD5=$SNUBA_MIGRATIONS_MD5" >> $GITHUB_OUTPUT
67-
6859
- name: Restore DB Volumes Cache
6960
id: restore_cache
7061
uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd
7162
with:
72-
key: db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}
63+
key: db-volumes-v6-${{ env.LATEST_TAG }}
7364
restore-keys: |
74-
db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}
65+
db-volumes-v6-${{ env.LATEST_TAG }}
7566
db-volumes-v6-
7667
volumes: |
7768
sentry-postgres
@@ -80,7 +71,10 @@ jobs:
8071
8172
- name: Install ${{ env.LATEST_TAG }}
8273
env:
74+
# Remove SKIP_DB_MIGRATIONS after releasing 25.1.1 or 25.2.0
8375
SKIP_DB_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }}
76+
SKIP_SENTRY_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }}
77+
SKIP_SNUBA_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }}
8478
run: |
8579
# This is to compensate for a bug in upgrade-clickhouse where
8680
# if we have sentry-clickhouse volume without the rest, it fails

‎action.yaml

Lines changed: 50 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -54,34 +54,58 @@ runs:
5454
sudo curl -L https://github.com/docker/compose/releases/download/${{ env.COMPOSE_VERSION }}/docker-compose-`uname -s`-`uname -m` -o "${{ env.COMPOSE_PATH }}/docker-compose"
5555
sudo chmod +x "${{ env.COMPOSE_PATH }}/docker-compose"
5656
57-
- name: Compute Docker Volume Cache Key
57+
- name: Compute Docker Volume Cache Keys
5858
id: cache_key
5959
shell: bash
6060
run: |
6161
source ${{ github.action_path }}/.env
6262
# See https://explainshell.com/explain?cmd=ls%20-Rv1rpq
6363
# for that long `ls` command
64-
SENTRY_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SENTRY_IMAGE -c 'ls -Rv1rpq src/sentry/migrations/' | md5sum | cut -d ' ' -f 1)
64+
SENTRY_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SENTRY_IMAGE -c '{ ls -Rv1rpq src/sentry/migrations/; sed -n "/KAFKA_TOPIC_TO_CLUSTER/,/}/p" src/sentry/conf/server.py; }' | md5sum | cut -d ' ' -f 1)
6565
echo "SENTRY_MIGRATIONS_MD5=$SENTRY_MIGRATIONS_MD5" >> $GITHUB_OUTPUT
66-
SNUBA_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SNUBA_IMAGE -c 'ls -Rv1rpq snuba/snuba_migrations/**/*.py' | md5sum | cut -d ' ' -f 1)
66+
SNUBA_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SNUBA_IMAGE -c '{ ls -Rv1rpq snuba/snuba_migrations/**/*.py; sed -n "/^class Topic(Enum):/,/\\n\\n/p" snuba/utils/streams/topics.py; }' | md5sum | cut -d ' ' -f 1)
6767
echo "SNUBA_MIGRATIONS_MD5=$SNUBA_MIGRATIONS_MD5" >> $GITHUB_OUTPUT
6868
69-
- name: Restore DB Volumes Cache
70-
id: restore_cache
69+
- name: Restore Sentry Volume Cache
70+
id: restore_cache_sentry
7171
uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd
7272
with:
73-
key: db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}
73+
key: db-volumes-sentry-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}
7474
restore-keys: |
75-
db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}
76-
db-volumes-v6-
75+
db-volumes-sentry-v1-
7776
volumes: |
7877
sentry-postgres
78+
79+
- name: Restore Snuba Volume Cache
80+
id: restore_cache_snuba
81+
uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd
82+
with:
83+
key: db-volumes-snuba-v1-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}
84+
restore-keys: |
85+
db-volumes-snuba-v1-
86+
volumes: |
7987
sentry-clickhouse
88+
89+
- name: Restore Kafka Volume Cache
90+
id: restore_cache_kafka
91+
uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd
92+
with:
93+
key: db-volumes-kafka-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}
94+
restore-keys: |
95+
db-volumes-kafka-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}
96+
db-volumes-kafka-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}-
97+
db-volumes-kafka-v1-
98+
volumes: |
8099
sentry-kafka
81100
82101
- name: Install self-hosted
83102
env:
84-
SKIP_DB_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }}
103+
# Note that cache keys for Sentry and Snuba have their respective Kafka configs built into them
104+
# and the Kafka volume cache is comprises both keys. This way we can omit the Kafka cache hit
105+
# in here to still avoid running Sentry or Snuba migrations if only one of their Kafka config has
106+
# changed. Heats up your head a bit but if you think about it, it makes sense.
107+
SKIP_SENTRY_MIGRATIONS: ${{ steps.restore_cache_sentry.outputs.cache-hit == 'true' && '1' || '' }}
108+
SKIP_SNUBA_MIGRATIONS: ${{ steps.restore_cache_snuba.outputs.cache-hit == 'true' && '1' || '' }}
85109
shell: bash
86110
run: |
87111
cd ${{ github.action_path }}
@@ -97,14 +121,28 @@ runs:
97121
98122
./install.sh --no-report-self-hosted-issues --skip-commit-check
99123
100-
- name: Save DB Volumes Cache
101-
if: steps.restore_cache.outputs.cache-hit != 'true'
124+
- name: Save Sentry Volume Cache
125+
if: steps.restore_cache_sentry.outputs.cache-hit != 'true'
102126
uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd
103127
with:
104-
key: ${{ steps.restore_cache.outputs.cache-primary-key }}
128+
key: ${{ steps.restore_cache_sentry.outputs.cache-primary-key }}
105129
volumes: |
106130
sentry-postgres
131+
132+
- name: Save Snuba Volume Cache
133+
if: steps.restore_cache_snuba.outputs.cache-hit != 'true'
134+
uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd
135+
with:
136+
key: ${{ steps.restore_cache_snuba.outputs.cache-primary-key }}
137+
volumes: |
107138
sentry-clickhouse
139+
140+
- name: Save Kafka Volume Cache
141+
if: steps.restore_cache_kafka.outputs.cache-hit != 'true'
142+
uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd
143+
with:
144+
key: ${{ steps.restore_cache_kafka.outputs.cache-primary-key }}
145+
volumes: |
108146
sentry-kafka
109147
110148
- name: Integration Test

‎install/bootstrap-snuba.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
echo "${_group}Bootstrapping and migrating Snuba ..."
22

3-
if [[ -z "${SKIP_DB_MIGRATIONS:-}" ]]; then
3+
if [[ -z "${SKIP_SNUBA_MIGRATIONS:-}" ]]; then
44
$dcr snuba-api bootstrap --force
55
else
6-
echo "Skipped DB migrations due to SKIP_DB_MIGRATIONS=$SKIP_DB_MIGRATIONS"
6+
echo "Skipped DB migrations due to SKIP_SNUBA_MIGRATIONS=$SKIP_SNUBA_MIGRATIONS"
77
fi
88

99
echo "${_endgroup}"

‎install/set-up-and-migrate-database.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
echo "${_group}Setting up / migrating database ..."
22

3-
if [[ -z "${SKIP_DB_MIGRATIONS:-}" ]]; then
3+
if [[ -z "${SKIP_SENTRY_MIGRATIONS:-}" ]]; then
44
# Fixes https://github.com/getsentry/self-hosted/issues/2758, where a migration fails due to indexing issue
55
$dc up --wait postgres
66

@@ -31,6 +31,6 @@ with connection.cursor() as cursor:
3131
$dcr web upgrade --create-kafka-topics
3232
fi
3333
else
34-
echo "Skipped DB migrations due to SKIP_DB_MIGRATIONS=$SKIP_DB_MIGRATIONS"
34+
echo "Skipped DB migrations due to SKIP_SENTRY_MIGRATIONS=$SKIP_SENTRY_MIGRATIONS"
3535
fi
3636
echo "${_endgroup}"

0 commit comments

Comments
 (0)