Skip to content

Commit a11ebc0

Browse files
tellet-qtellet-q
authored
Add continuous benchmark for tenants (#183)
1 parent 4ffc3ce commit a11ebc0

File tree

9 files changed

+240
-29
lines changed

9 files changed

+240
-29
lines changed

‎.github/workflows/continuous-benchmark.yaml

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,20 @@ jobs:
4444
timeout 30m bash -x tools/run_ci.sh
4545
done
4646
47+
# Benchmark filtered search by tenants with mem limitation
48+
49+
export ENGINE_NAME="qdrant-all-on-disk-scalar-q"
50+
export DATASETS="random-768-100-tenants"
51+
export CONTAINER_MEM_LIMIT=150mb
52+
53+
# Benchmark the dev branch:
54+
export QDRANT_VERSION=ghcr/dev
55+
timeout 30m bash -x tools/run_ci.sh
56+
57+
# Benchmark the master branch:
58+
export QDRANT_VERSION=docker/master
59+
timeout 30m bash -x tools/run_ci.sh
60+
4761
set -e
4862
- name: Fail job if any of the benches failed
4963
if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout'
@@ -67,4 +81,4 @@ jobs:
6781
}
6882
env:
6983
SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }}
70-
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
84+
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

‎datasets/datasets.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,17 @@
336336
"b": "keyword"
337337
}
338338
},
339+
{
340+
"name": "random-768-100-tenants",
341+
"vector_size": 768,
342+
"distance": "cosine",
343+
"type": "tar",
344+
"link": "https://storage.googleapis.com/ann-filtered-benchmark/datasets/random_keywords_1m_768_vocab_100.tgz",
345+
"path": "random-768-100-tenants/random_keywords_1m_768_vocab_100",
346+
"schema": {
347+
"a": "keyword"
348+
}
349+
},
339350
{
340351
"name": "random-100-match-kw-small-vocab-no-filters",
341352
"vector_size": 256,

‎engine/clients/qdrant/configure.py

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,13 @@ class QdrantConfigurator(BaseConfigurator):
2121
"float": rest.PayloadSchemaType.FLOAT,
2222
"geo": rest.PayloadSchemaType.GEO,
2323
}
24+
INDEX_PARAMS_TYPE_MAPPING = {
25+
"int": rest.IntegerIndexParams,
26+
"keyword": rest.KeywordIndexParams,
27+
"text": rest.TextIndexParams,
28+
"float": rest.FloatIndexParams,
29+
"geo": rest.GeoIndexParams,
30+
}
2431

2532
def __init__(self, host, collection_params: dict, connection_params: dict):
2633
super().__init__(host, collection_params, connection_params)
@@ -43,15 +50,25 @@ def recreate(self, dataset: Dataset, collection_params):
4350
},
4451
}
4552
else:
53+
is_vectors_on_disk = self.collection_params.get("vectors_config", {}).get(
54+
"on_disk", False
55+
)
56+
self.collection_params.pop("vectors_config", None)
57+
4658
vectors_config = {
4759
"vectors_config": (
4860
rest.VectorParams(
4961
size=dataset.config.vector_size,
5062
distance=self.DISTANCE_MAPPING.get(dataset.config.distance),
63+
on_disk=is_vectors_on_disk,
5164
)
5265
)
5366
}
5467

68+
payload_index_params = self.collection_params.pop("payload_index_params", {})
69+
if not set(payload_index_params.keys()).issubset(dataset.config.schema.keys()):
70+
raise ValueError("payload_index_params are not found in dataset schema")
71+
5572
self.client.recreate_collection(
5673
collection_name=QDRANT_COLLECTION_NAME,
5774
**vectors_config,
@@ -65,8 +82,24 @@ def recreate(self, dataset: Dataset, collection_params):
6582
),
6683
)
6784
for field_name, field_type in dataset.config.schema.items():
68-
self.client.create_payload_index(
69-
collection_name=QDRANT_COLLECTION_NAME,
70-
field_name=field_name,
71-
field_schema=self.INDEX_TYPE_MAPPING.get(field_type),
72-
)
85+
if field_type in ["keyword", "uuid"]:
86+
is_tenant = payload_index_params.get(field_name, {}).get(
87+
"is_tenant", None
88+
)
89+
on_disk = payload_index_params.get(field_name, {}).get("on_disk", None)
90+
91+
self.client.create_payload_index(
92+
collection_name=QDRANT_COLLECTION_NAME,
93+
field_name=field_name,
94+
field_schema=self.INDEX_PARAMS_TYPE_MAPPING.get(field_type)(
95+
type=self.INDEX_TYPE_MAPPING.get(field_type),
96+
is_tenant=is_tenant,
97+
on_disk=on_disk,
98+
),
99+
)
100+
else:
101+
self.client.create_payload_index(
102+
collection_name=QDRANT_COLLECTION_NAME,
103+
field_name=field_name,
104+
field_schema=self.INDEX_TYPE_MAPPING.get(field_type),
105+
)

‎engine/servers/qdrant-continuous-benchmarks-with-volume/docker-compose.yaml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
version: '3.7'
2+
3+
services:
4+
qdrant_bench:
5+
image: ${CONTAINER_REGISTRY:-docker.io}/qdrant/qdrant:${QDRANT_VERSION}
6+
container_name: qdrant-continuous
7+
ports:
8+
- "6333:6333"
9+
- "6334:6334"
10+
volumes:
11+
- qdrant_storage:/qdrant/storage
12+
logging:
13+
driver: "json-file"
14+
options:
15+
max-file: 1
16+
max-size: 10m
17+
deploy:
18+
resources:
19+
limits:
20+
memory: ${CONTAINER_MEM_LIMIT:-25Gb}
21+
22+
volumes:
23+
qdrant_storage:
24+
name: "qdrant_storage"
25+
driver: local
26+
driver_opts:
27+
type: none
28+
device: ${PWD}/qdrant_storage
29+
o: bind

‎experiments/configurations/qdrant-on-disk.json

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,24 @@
1111
{ "parallel": 8, "config": { "hnsw_ef": 128 } }
1212
],
1313
"upload_params": { "parallel": 4 }
14+
},
15+
{
16+
"name": "qdrant-all-on-disk-scalar-q",
17+
"engine": "qdrant",
18+
"connection_params": {},
19+
"collection_params": {
20+
"optimizers_config": { "default_segment_number": 17 },
21+
"quantization_config": { "scalar": {"type": "int8", "quantile": 0.99, "always_ram": false} },
22+
"vectors_config": { "on_disk": true },
23+
"hnsw_config": { "on_disk": true, "m": 0, "payload_m": 16 },
24+
"on_disk_payload": true,
25+
"payload_index_params": {
26+
"a": { "is_tenant": true, "on_disk": true }
27+
}
28+
},
29+
"search_params": [
30+
{ "parallel": 8 }
31+
],
32+
"upload_params": { "parallel": 4 }
1433
}
1534
]

‎tools/run_client_script.sh

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
PS4='ts=$(date "+%Y-%m-%dT%H:%M:%SZ") level=DEBUG line=$LINENO file=$BASH_SOURCE '
44
set -euo pipefail
55

6+
# Possible values are: full|upload|search
7+
EXPERIMENT_MODE=${1:-"full"}
8+
69
CLOUD_NAME=${CLOUD_NAME:-"hetzner"}
710
SERVER_USERNAME=${SERVER_USERNAME:-"root"}
811

@@ -22,16 +25,26 @@ DATASETS=${DATASETS:-"laion-small-clip"}
2225

2326
PRIVATE_IP_OF_THE_SERVER=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_private_ip.sh" "$BENCH_SERVER_NAME")
2427

25-
RUN_EXPERIMENT="ENGINE_NAME=${ENGINE_NAME} DATASETS=${DATASETS} PRIVATE_IP_OF_THE_SERVER=${PRIVATE_IP_OF_THE_SERVER} bash ~/run_experiment.sh"
28+
RUN_EXPERIMENT="ENGINE_NAME=${ENGINE_NAME} DATASETS=${DATASETS} PRIVATE_IP_OF_THE_SERVER=${PRIVATE_IP_OF_THE_SERVER} EXPERIMENT_MODE=${EXPERIMENT_MODE} bash ~/run_experiment.sh"
2629

2730
ssh -tt -o ServerAliveInterval=60 -o ServerAliveCountMax=3 "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "${RUN_EXPERIMENT}"
2831

29-
SEARCH_RESULT_FILE=$(ssh "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "ls -t results/*-search-*.json | head -n 1")
30-
UPLOAD_RESULT_FILE=$(ssh "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "ls -t results/*-upload-*.json | head -n 1")
32+
echo "Gather experiment results..."
33+
result_files_arr=()
34+
35+
if [[ "$EXPERIMENT_MODE" == "full" ]] || [[ "$EXPERIMENT_MODE" == "upload" ]]; then
36+
UPLOAD_RESULT_FILE=$(ssh "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "ls -t results/*-upload-*.json | head -n 1")
37+
result_files_arr+=("$UPLOAD_RESULT_FILE")
38+
fi
39+
40+
if [[ "$EXPERIMENT_MODE" == "full" ]] || [[ "$EXPERIMENT_MODE" == "search" ]]; then
41+
SEARCH_RESULT_FILE=$(ssh "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "ls -t results/*-search-*.json | head -n 1")
42+
result_files_arr+=("$SEARCH_RESULT_FILE")
43+
fi
3144

3245
mkdir -p results
3346

34-
for RESULT_FILE in $SEARCH_RESULT_FILE $UPLOAD_RESULT_FILE; do
47+
for RESULT_FILE in "${result_files_arr[@]}"; do
3548
# -p preseves modification time, access time, and modes (but not change time)
3649
scp -p "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/${RESULT_FILE}" "./results"
3750
done

‎tools/run_experiment.sh

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ DATASETS=${DATASETS:-""}
99

1010
PRIVATE_IP_OF_THE_SERVER=${PRIVATE_IP_OF_THE_SERVER:-""}
1111

12+
EXPERIMENT_MODE=${EXPERIMENT_MODE:-"full"}
13+
1214
if [[ -z "$ENGINE_NAME" ]]; then
1315
echo "ENGINE_NAME is not set"
1416
exit 1
@@ -24,23 +26,40 @@ if [[ -z "$PRIVATE_IP_OF_THE_SERVER" ]]; then
2426
exit 1
2527
fi
2628

29+
if [[ -z "$EXPERIMENT_MODE" ]]; then
30+
echo "EXPERIMENT_MODE is not set, possible values are: full | upload | search"
31+
exit 1
32+
fi
2733
docker container rm -f ci-benchmark-upload || true
2834
docker container rm -f ci-benchmark-search || true
2935

3036
docker rmi --force qdrant/vector-db-benchmark:latest || true
3137

32-
docker run \
33-
--rm \
34-
-it \
35-
--name ci-benchmark-upload \
36-
-v "$HOME/results:/code/results" \
37-
qdrant/vector-db-benchmark:latest \
38-
python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-search
39-
40-
docker run \
41-
--rm \
42-
-it \
43-
--name ci-benchmark-search \
44-
-v "$HOME/results:/code/results" \
45-
qdrant/vector-db-benchmark:latest \
46-
python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-upload
38+
if [[ "$EXPERIMENT_MODE" == "full" ]] || [[ "$EXPERIMENT_MODE" == "upload" ]]; then
39+
echo "EXPERIMENT_MODE=$EXPERIMENT_MODE"
40+
docker run \
41+
--rm \
42+
-it \
43+
--name ci-benchmark-upload \
44+
-v "$HOME/results:/code/results" \
45+
qdrant/vector-db-benchmark:latest \
46+
python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-search
47+
fi
48+
49+
50+
if [[ "$EXPERIMENT_MODE" == "full" ]] || [[ "$EXPERIMENT_MODE" == "search" ]]; then
51+
echo "EXPERIMENT_MODE=$EXPERIMENT_MODE"
52+
53+
if [[ "$EXPERIMENT_MODE" == "search" ]]; then
54+
echo "Drop caches before running the experiment"
55+
sudo bash -c 'sync; echo 1 > /proc/sys/vm/drop_caches'
56+
fi
57+
58+
docker run \
59+
--rm \
60+
-it \
61+
--name ci-benchmark-search \
62+
-v "$HOME/results:/code/results" \
63+
qdrant/vector-db-benchmark:latest \
64+
python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-upload
65+
fi

‎tools/run_remote_benchmark.sh

100644100755
Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,31 @@ trap 'cleanup' EXIT
3434
SERVER_NAME=$BENCH_SERVER_NAME bash -x "${SCRIPT_PATH}/${CLOUD_NAME}/check_ssh_connection.sh"
3535
SERVER_NAME=$BENCH_CLIENT_NAME bash -x "${SCRIPT_PATH}/${CLOUD_NAME}/check_ssh_connection.sh"
3636

37+
if [[ -z "${CONTAINER_MEM_LIMIT:-}" ]]; then
38+
echo "CONTAINER_MEM_LIMIT is not set, run without memory limit"
3739

38-
SERVER_CONTAINER_NAME=${SERVER_CONTAINER_NAME:-"qdrant-continuous-benchmarks"}
40+
SERVER_CONTAINER_NAME=${SERVER_CONTAINER_NAME:-"qdrant-continuous-benchmarks"}
3941

40-
bash -x "${SCRIPT_PATH}/run_server_container.sh" "$SERVER_CONTAINER_NAME"
42+
bash -x "${SCRIPT_PATH}/run_server_container.sh" "$SERVER_CONTAINER_NAME"
4143

42-
bash -x "${SCRIPT_PATH}/run_client_script.sh"
44+
bash -x "${SCRIPT_PATH}/run_client_script.sh"
45+
46+
bash -x "${SCRIPT_PATH}/qdrant_collect_stats.sh" "$SERVER_CONTAINER_NAME"
47+
48+
else
49+
echo "CONTAINER_MEM_LIMIT is set, run search with memory limit: ${CONTAINER_MEM_LIMIT}"
50+
51+
SERVER_CONTAINER_NAME=${SERVER_CONTAINER_NAME:-"qdrant-continuous-benchmarks-with-volume"}
52+
53+
bash -x "${SCRIPT_PATH}/run_server_container_with_volume.sh" "$SERVER_CONTAINER_NAME"
54+
55+
bash -x "${SCRIPT_PATH}/run_client_script.sh" "upload"
56+
57+
bash -x "${SCRIPT_PATH}/run_server_container_with_volume.sh" "$SERVER_CONTAINER_NAME" "$CONTAINER_MEM_LIMIT" "continue"
58+
59+
bash -x "${SCRIPT_PATH}/run_client_script.sh" "search"
60+
61+
bash -x "${SCRIPT_PATH}/qdrant_collect_stats.sh" "$SERVER_CONTAINER_NAME"
62+
63+
fi
4364

44-
bash -x "${SCRIPT_PATH}/qdrant_collect_stats.sh" "$SERVER_CONTAINER_NAME"

‎tools/run_server_container_with_volume.sh

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
# Examples: qdrant-continuous-benchmarks-with-volume
6+
CONTAINER_NAME=$1
7+
CONTAINER_MEM_LIMIT=${2:-"25Gb"}
8+
EXECUTION_MODE=${3:-"init"}
9+
10+
CLOUD_NAME=${CLOUD_NAME:-"hetzner"}
11+
SERVER_USERNAME=${SERVER_USERNAME:-"root"}
12+
13+
14+
SCRIPT=$(realpath "$0")
15+
SCRIPT_PATH=$(dirname "$SCRIPT")
16+
17+
BENCH_SERVER_NAME=${SERVER_NAME:-"benchmark-server-1"}
18+
19+
QDRANT_VERSION=${QDRANT_VERSION:-"dev"}
20+
21+
IP_OF_THE_SERVER=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_public_ip.sh" "$BENCH_SERVER_NAME")
22+
23+
bash -x "${SCRIPT_PATH}/sync_servers.sh" "root@$IP_OF_THE_SERVER"
24+
25+
# if version is starts with "docker" or "ghcr", use container
26+
if [[ ${QDRANT_VERSION} == docker/* ]] || [[ ${QDRANT_VERSION} == ghcr/* ]]; then
27+
28+
if [[ ${QDRANT_VERSION} == docker/* ]]; then
29+
# pull from docker hub
30+
QDRANT_VERSION=${QDRANT_VERSION#docker/}
31+
CONTAINER_REGISTRY='docker.io'
32+
elif [[ ${QDRANT_VERSION} == ghcr/* ]]; then
33+
# pull from github container registry
34+
QDRANT_VERSION=${QDRANT_VERSION#ghcr/}
35+
CONTAINER_REGISTRY='ghcr.io'
36+
fi
37+
38+
if [[ "$EXECUTION_MODE" == "init" ]]; then
39+
# create volume qdrant_storage
40+
echo "Initialize qdrant from scratch"
41+
DOCKER_VOLUME_SET_UP="docker volume rm -f qdrant_storage; sudo rm -rf qdrant_storage; mkdir qdrant_storage"
42+
DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export CONTAINER_MEM_LIMIT=${CONTAINER_MEM_LIMIT}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true ; ${DOCKER_VOLUME_SET_UP}; docker compose up -d; docker container ls -a"
43+
else
44+
# suggest that volume qdrant_storage exist and start qdrant
45+
echo "Reload qdrant with existing data"
46+
DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; export CONTAINER_REGISTRY=${CONTAINER_REGISTRY}; export CONTAINER_MEM_LIMIT=${CONTAINER_MEM_LIMIT}; docker compose down; pkill qdrant; docker rm -f qdrant-continuous || true; docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true ; sudo bash -c 'sync; echo 1 > /proc/sys/vm/drop_caches'; docker compose up -d; docker container ls -a"
47+
fi
48+
49+
ssh -t -o ServerAliveInterval=60 -o ServerAliveCountMax=3 "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "cd ./projects/vector-db-benchmark/engine/servers/${CONTAINER_NAME} ; $DOCKER_COMPOSE"
50+
else
51+
echo "Error: unknown version ${QDRANT_VERSION}. Version name should start with 'docker/' or 'ghcr/'"
52+
exit 1
53+
fi

0 commit comments

Comments
 (0)