Skip to content

Commit f4cb791

Browse files
tellet-qtellet-q
committed
Split into 2 and introduce concurrency groups
1 parent bb482e1 commit f4cb791

File tree

2 files changed

+126
-107
lines changed

2 files changed

+126
-107
lines changed

‎.github/workflows/continuous-benchmark-2.yaml

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
name: Continuous Benchmark
2+
3+
on:
4+
repository_dispatch:
5+
workflow_dispatch:
6+
schedule:
7+
# Run every day at midnight
8+
- cron: "0 0 * * *"
9+
10+
# Restrict to only running this workflow one at a time.
11+
# Any new runs will be queued until the previous run is complete.
12+
# Any existing pending runs will be cancelled and replaced with current run.
13+
concurrency:
14+
group: continuous-benchmark
15+
16+
jobs:
17+
# Schedule this benchmark to run once a day for the sake of saving on S3 costs.
18+
runLoadTimeBenchmark:
19+
runs-on: ubuntu-latest
20+
steps:
21+
- uses: actions/checkout@v3
22+
- uses: webfactory/[email protected]
23+
with:
24+
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
25+
- name: Benches
26+
id: benches
27+
run: |
28+
export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
29+
export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
30+
export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
31+
export SERVER_NAME="benchmark-server-3"
32+
bash -x tools/setup_ci.sh
33+
34+
set +e
35+
36+
# Benchmark collection load time
37+
export BENCHMARK_STRATEGY="collection-reload"
38+
39+
declare -A DATASET_TO_ENGINE
40+
declare -A DATASET_TO_URL
41+
DATASET_TO_ENGINE["all-payloads-default"]="qdrant-continuous-benchmark-snapshot"
42+
DATASET_TO_ENGINE["all-payloads-on-disk"]="qdrant-continuous-benchmark-snapshot"
43+
DATASET_TO_ENGINE["all-payloads-default-sparse"]="qdrant-continuous-benchmark-snapshot"
44+
DATASET_TO_ENGINE["all-payloads-on-disk-sparse"]="qdrant-continuous-benchmark-snapshot"
45+
46+
export STORAGE_URL="https://storage.googleapis.com/qdrant-benchmark-snapshots/all-payloads"
47+
DATASET_TO_URL["all-payloads-default"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-default.snapshot"
48+
DATASET_TO_URL["all-payloads-on-disk"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-on-disk.snapshot"
49+
DATASET_TO_URL["all-payloads-default-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-default.snapshot"
50+
DATASET_TO_URL["all-payloads-on-disk-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-on-disk.snapshot"
51+
52+
set +e
53+
54+
for dataset in "${!DATASET_TO_ENGINE[@]}"; do
55+
export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]}
56+
export DATASETS=$dataset
57+
export SNAPSHOT_URL=${DATASET_TO_URL[$dataset]}
58+
59+
# Benchmark the dev branch:
60+
export QDRANT_VERSION=ghcr/dev
61+
timeout 30m bash -x tools/run_ci.sh
62+
63+
# Benchmark the master branch:
64+
export QDRANT_VERSION=docker/master
65+
timeout 30m bash -x tools/run_ci.sh
66+
done
67+
68+
set -e
69+
- name: Fail job if any of the benches failed
70+
if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout'
71+
run: exit 1
72+
- name: Send Notification
73+
if: failure() || cancelled()
74+
uses: slackapi/[email protected]
75+
with:
76+
payload: |
77+
{
78+
"text": "CI benchmarks (runLoadTimeBenchmark) run status: ${{ job.status }}",
79+
"blocks": [
80+
{
81+
"type": "section",
82+
"text": {
83+
"type": "mrkdwn",
84+
"text": "CI benchmarks (runLoadTimeBenchmark) failed because of *${{ steps.benches.outputs.failed }}*."
85+
}
86+
},
87+
{
88+
"type": "section",
89+
"text": {
90+
"type": "mrkdwn",
91+
"text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*."
92+
}
93+
},
94+
{
95+
"type": "section",
96+
"text": {
97+
"type": "mrkdwn",
98+
"text": "Engine: *${{ steps.benches.outputs.engine_name }}*."
99+
}
100+
},
101+
{
102+
"type": "section",
103+
"text": {
104+
"type": "mrkdwn",
105+
"text": "Dataset: *${{ steps.benches.outputs.dataset }}*."
106+
}
107+
},
108+
{
109+
"type": "section",
110+
"text": {
111+
"type": "mrkdwn",
112+
"text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"
113+
}
114+
}
115+
]
116+
}
117+
env:
118+
SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }}
119+
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

‎.github/workflows/continuous-benchmark.yaml

Lines changed: 7 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@ on:
77
# Run every 4 hours
88
- cron: "0 */4 * * *"
99

10+
# Restrict to only running this workflow one at a time.
11+
# Any new runs will be queued until the previous run is complete.
12+
# Any existing pending runs will be cancelled and replaced with current run.
13+
concurrency:
14+
group: continuous-benchmark
15+
1016
jobs:
1117
runBenchmark:
1218
runs-on: ubuntu-latest
@@ -182,115 +188,9 @@ jobs:
182188
env:
183189
SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }}
184190
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
185-
# Schedule this benchmark to run once a day for the sake of saving on S3 costs.
186-
runLoadTimeBenchmark:
187-
runs-on: ubuntu-latest
188-
needs: runBenchmark
189-
if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' || github.event.schedule == '0 0 * * *') }}
190-
steps:
191-
- uses: actions/checkout@v3
192-
- uses: webfactory/[email protected]
193-
with:
194-
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
195-
- name: Benches
196-
id: benches
197-
run: |
198-
export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
199-
export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
200-
export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
201-
export SERVER_NAME="benchmark-server-3"
202-
bash -x tools/setup_ci.sh
203-
204-
set +e
205-
206-
# Benchmark collection load time
207-
export BENCHMARK_STRATEGY="collection-reload"
208-
209-
declare -A DATASET_TO_ENGINE
210-
declare -A DATASET_TO_URL
211-
DATASET_TO_ENGINE["all-payloads-default"]="qdrant-continuous-benchmark-snapshot"
212-
DATASET_TO_ENGINE["all-payloads-on-disk"]="qdrant-continuous-benchmark-snapshot"
213-
DATASET_TO_ENGINE["all-payloads-default-sparse"]="qdrant-continuous-benchmark-snapshot"
214-
DATASET_TO_ENGINE["all-payloads-on-disk-sparse"]="qdrant-continuous-benchmark-snapshot"
215-
216-
export STORAGE_URL="https://storage.googleapis.com/qdrant-benchmark-snapshots/all-payloads"
217-
DATASET_TO_URL["all-payloads-default"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-default.snapshot"
218-
DATASET_TO_URL["all-payloads-on-disk"]="${STORAGE_URL}/benchmark-all-payloads-500k-768-on-disk.snapshot"
219-
DATASET_TO_URL["all-payloads-default-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-default.snapshot"
220-
DATASET_TO_URL["all-payloads-on-disk-sparse"]="${STORAGE_URL}/benchmark-all-payloads-500k-sparse-on-disk.snapshot"
221-
222-
set +e
223-
224-
for dataset in "${!DATASET_TO_ENGINE[@]}"; do
225-
export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]}
226-
export DATASETS=$dataset
227-
export SNAPSHOT_URL=${DATASET_TO_URL[$dataset]}
228-
229-
# Benchmark the dev branch:
230-
export QDRANT_VERSION=ghcr/dev
231-
timeout 30m bash -x tools/run_ci.sh
232-
233-
# Benchmark the master branch:
234-
export QDRANT_VERSION=docker/master
235-
timeout 30m bash -x tools/run_ci.sh
236-
done
237-
238-
set -e
239-
- name: Fail job if any of the benches failed
240-
if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout'
241-
run: exit 1
242-
- name: Send Notification
243-
if: failure() || cancelled()
244-
uses: slackapi/[email protected]
245-
with:
246-
payload: |
247-
{
248-
"text": "CI benchmarks (runLoadTimeBenchmark) run status: ${{ job.status }}",
249-
"blocks": [
250-
{
251-
"type": "section",
252-
"text": {
253-
"type": "mrkdwn",
254-
"text": "CI benchmarks (runLoadTimeBenchmark) failed because of *${{ steps.benches.outputs.failed }}*."
255-
}
256-
},
257-
{
258-
"type": "section",
259-
"text": {
260-
"type": "mrkdwn",
261-
"text": "Qdrant version: *${{ steps.benches.outputs.qdrant_version }}*."
262-
}
263-
},
264-
{
265-
"type": "section",
266-
"text": {
267-
"type": "mrkdwn",
268-
"text": "Engine: *${{ steps.benches.outputs.engine_name }}*."
269-
}
270-
},
271-
{
272-
"type": "section",
273-
"text": {
274-
"type": "mrkdwn",
275-
"text": "Dataset: *${{ steps.benches.outputs.dataset }}*."
276-
}
277-
},
278-
{
279-
"type": "section",
280-
"text": {
281-
"type": "mrkdwn",
282-
"text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"
283-
}
284-
}
285-
]
286-
}
287-
env:
288-
SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }}
289-
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
290-
291191
runParallelBenchmark:
292192
runs-on: ubuntu-latest
293-
needs: [ runLoadTimeBenchmark, runTenantsBenchmark ]
193+
needs: runTenantsBenchmark
294194
if: ${{ always() }}
295195
steps:
296196
- uses: actions/checkout@v3

0 commit comments

Comments
 (0)