Skip to content

Commit 5ec825e

Browse files
generallgenerall
authored
Merge pull request #218 from qdrant/ci/reduce-snapshots-bench-runs
Only run runLoadTimeBenchmark once per day
2 parents a26483b + 6bc2439 commit 5ec825e

File tree

2 files changed

+126
-106
lines changed

2 files changed

+126
-106
lines changed

‎.github/workflows/continuous-benchmark-2.yaml

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
name: Continuous Benchmark 2
2+
3+
on:
4+
repository_dispatch:
5+
workflow_dispatch:
6+
schedule:
7+
# Run every day at midnight
8+
- cron: "0 0 * * *"
9+
10+
# Restrict to only running this workflow one at a time.
11+
# Any new runs will be queued until the previous run is complete.
12+
# Any existing pending runs will be cancelled and replaced with current run.
13+
concurrency:
14+
group: continuous-benchmark
15+
16+
jobs:
17+
# Schedule this benchmark to run once a day for the sake of saving on S3 costs.
18+
runLoadTimeBenchmark:
19+
runs-on: ubuntu-latest
20+
steps:
21+
- uses: actions/checkout@v3
22+
- uses: webfactory/[email protected]
23+
with:
24+
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
25+
- name: Benches
26+
id: benches
27+
run: |
28+
export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
29+
export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
30+
export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
31+
export SERVER_NAME="benchmark-server-3"
32+
bash -x tools/setup_ci.sh
33+
34+
set +e
35+
36+
# Benchmark collection load time
37+
export BENCHMARK_STRATEGY="collection-reload"
38+
39+
declare -A DATASET_TO_ENGINE
40+
declare -A DATASET_TO_URL
41+
DATASET_TO_ENGINE["all-payloads-default"]="qdrant-continuous-benchmark-snapshot"
42+
DATASET_TO_ENGINE["all-payloads-on-disk"]="qdrant-continuous-benchmark-snapshot"
43+
DATASET_TO_ENGINE["all-payloads-default-sparse"]="qdrant-continuous-benchmark-snapshot"
44+
DATASET_TO_ENGINE["all-payloads-on-disk-sparse"]="qdrant-continuous-benchmark-snapshot"
45+
46+
export STORAGE_URL="https://storage.googleapis.com/qdrant-benchmark-snapshots/all-payloads"
47+
DATASET_TO_URL["all-payloads-default"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-default.snapshot"
48+
DATASET_TO_URL["all-payloads-on-disk"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-on-disk.snapshot"
49+
DATASET_TO_URL["all-payloads-default-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-default.snapshot"
50+
DATASET_TO_URL["all-payloads-on-disk-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-on-disk.snapshot"
51+
52+
set +e
53+
54+
for dataset in "${!DATASET_TO_ENGINE[@]}"; do
55+
export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]}
56+
export DATASETS=$dataset
57+
export SNAPSHOT_URL=${DATASET_TO_URL[$dataset]}
58+
59+
# Benchmark the dev branch:
60+
export QDRANT_VERSION=ghcr/dev
61+
timeout 30m bash -x tools/run_ci.sh
62+
63+
# Benchmark the master branch:
64+
export QDRANT_VERSION=docker/master
65+
timeout 30m bash -x tools/run_ci.sh
66+
done
67+
68+
set -e
69+
- name: Fail job if any of the benches failed
70+
if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout'
71+
run: exit 1
72+
- name: Send Notification
73+
if: failure() || cancelled()
74+
uses: slackapi/[email protected]
75+
with:
76+
payload: |
77+
{
78+
"text": "CI benchmarks (runLoadTimeBenchmark) run status: ${{ job.status }}",
79+
"blocks": [
80+
{
81+
"type": "section",
82+
"text": {
83+
"type": "mrkdwn",
84+
"text": "CI benchmarks (runLoadTimeBenchmark) failed because of *${{ steps.benches.outputs.failed }}*."
85+
}
86+
},
87+
{
88+
"type": "section",
89+
"text": {
90+
"type": "mrkdwn",
91+
"text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*."
92+
}
93+
},
94+
{
95+
"type": "section",
96+
"text": {
97+
"type": "mrkdwn",
98+
"text": "Engine: *${{ steps.benches.outputs.engine_name }}*."
99+
}
100+
},
101+
{
102+
"type": "section",
103+
"text": {
104+
"type": "mrkdwn",
105+
"text": "Dataset: *${{ steps.benches.outputs.dataset }}*."
106+
}
107+
},
108+
{
109+
"type": "section",
110+
"text": {
111+
"type": "mrkdwn",
112+
"text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"
113+
}
114+
}
115+
]
116+
}
117+
env:
118+
SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }}
119+
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

‎.github/workflows/continuous-benchmark.yaml

Lines changed: 7 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@ on:
77
# Run every 4 hours
88
- cron: "0 */4 * * *"
99

10+
# Restrict to only running this workflow one at a time.
11+
# Any new runs will be queued until the previous run is complete.
12+
# Any existing pending runs will be cancelled and replaced with current run.
13+
concurrency:
14+
group: continuous-benchmark
15+
1016
jobs:
1117
runBenchmark:
1218
runs-on: ubuntu-latest
@@ -182,114 +188,9 @@ jobs:
182188
env:
183189
SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }}
184190
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
185-
runLoadTimeBenchmark:
186-
runs-on: ubuntu-latest
187-
needs: runBenchmark
188-
if: ${{ always() }}
189-
steps:
190-
- uses: actions/checkout@v3
191-
- uses: webfactory/[email protected]
192-
with:
193-
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
194-
- name: Benches
195-
id: benches
196-
run: |
197-
export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
198-
export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
199-
export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
200-
export SERVER_NAME="benchmark-server-3"
201-
bash -x tools/setup_ci.sh
202-
203-
set +e
204-
205-
# Benchmark collection load time
206-
export BENCHMARK_STRATEGY="collection-reload"
207-
208-
declare -A DATASET_TO_ENGINE
209-
declare -A DATASET_TO_URL
210-
DATASET_TO_ENGINE["all-payloads-default"]="qdrant-continuous-benchmark-snapshot"
211-
DATASET_TO_ENGINE["all-payloads-on-disk"]="qdrant-continuous-benchmark-snapshot"
212-
DATASET_TO_ENGINE["all-payloads-default-sparse"]="qdrant-continuous-benchmark-snapshot"
213-
DATASET_TO_ENGINE["all-payloads-on-disk-sparse"]="qdrant-continuous-benchmark-snapshot"
214-
215-
export STORAGE_URL="https://storage.googleapis.com/qdrant-benchmark-snapshots/all-payloads"
216-
DATASET_TO_URL["all-payloads-default"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-default.snapshot"
217-
DATASET_TO_URL["all-payloads-on-disk"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-on-disk.snapshot"
218-
DATASET_TO_URL["all-payloads-default-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-default.snapshot"
219-
DATASET_TO_URL["all-payloads-on-disk-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-on-disk.snapshot"
220-
221-
set +e
222-
223-
for dataset in "${!DATASET_TO_ENGINE[@]}"; do
224-
export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]}
225-
export DATASETS=$dataset
226-
export SNAPSHOT_URL=${DATASET_TO_URL[$dataset]}
227-
228-
# Benchmark the dev branch:
229-
export QDRANT_VERSION=ghcr/dev
230-
timeout 30m bash -x tools/run_ci.sh
231-
232-
# Benchmark the master branch:
233-
export QDRANT_VERSION=docker/master
234-
timeout 30m bash -x tools/run_ci.sh
235-
done
236-
237-
set -e
238-
- name: Fail job if any of the benches failed
239-
if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout'
240-
run: exit 1
241-
- name: Send Notification
242-
if: failure() || cancelled()
243-
uses: slackapi/[email protected]
244-
with:
245-
payload: |
246-
{
247-
"text": "CI benchmarks (runLoadTimeBenchmark) run status: ${{ job.status }}",
248-
"blocks": [
249-
{
250-
"type": "section",
251-
"text": {
252-
"type": "mrkdwn",
253-
"text": "CI benchmarks (runLoadTimeBenchmark) failed because of *${{ steps.benches.outputs.failed }}*."
254-
}
255-
},
256-
{
257-
"type": "section",
258-
"text": {
259-
"type": "mrkdwn",
260-
"text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*."
261-
}
262-
},
263-
{
264-
"type": "section",
265-
"text": {
266-
"type": "mrkdwn",
267-
"text": "Engine: *${{ steps.benches.outputs.engine_name }}*."
268-
}
269-
},
270-
{
271-
"type": "section",
272-
"text": {
273-
"type": "mrkdwn",
274-
"text": "Dataset: *${{ steps.benches.outputs.dataset }}*."
275-
}
276-
},
277-
{
278-
"type": "section",
279-
"text": {
280-
"type": "mrkdwn",
281-
"text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"
282-
}
283-
}
284-
]
285-
}
286-
env:
287-
SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }}
288-
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
289-
290191
runParallelBenchmark:
291192
runs-on: ubuntu-latest
292-
needs: [ runLoadTimeBenchmark, runTenantsBenchmark ]
193+
needs: runTenantsBenchmark
293194
if: ${{ always() }}
294195
steps:
295196
- uses: actions/checkout@v3

0 commit comments

Comments
 (0)