Add delete-n-update benchmark (#253)

tellet-q · pre-commit-ci[bot] · web-flow · commit 9ae9839d5702 · 2025-05-28T12:17:10.000+02:00
* Add delete-n-update benchmark * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
diff --git a/.github/workflows/actions/create-inventory/action.yaml b/.github/workflows/actions/create-inventory/action.yaml
@@ -0,0 +1,41 @@
+name: Create inventory
+description: "Prepare inventory.ini"
+inputs:
+  hcloud_token:
+    description: "Hetzner Cloud API token"
+    required: true
+  server_name:
+    description: "Name of the server"
+    required: true
+  db_host:
+    description: "Database host"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create inventory.ini
+      shell: bash
+      run: |
+        export HCLOUD_TOKEN=${{ inputs.hcloud_token }}
+        export POSTGRES_HOST=${{ inputs.db_host }}
+        export SERVER_NAME="${{ inputs.server_name }}"
+
+        # Download and install hcloud
+        HCVERSION=v1.36.0
+        wget https://github.com/hetznercloud/cli/releases/download/${HCVERSION}/hcloud-linux-amd64.tar.gz
+        tar xzf hcloud-linux-amd64.tar.gz
+        mv hcloud /usr/local/bin
+        chmod +x /usr/local/bin/hcloud
+
+        IP_OF_THE_SERVER=$(bash "tools/hetzner/get_public_ip.sh" "$SERVER_NAME")
+
+        # Create ansible inventory.ini file
+        cat <<EOL > inventory.ini
+        [remote_machines]
+        benchmark-machine ansible_host=${IP_OF_THE_SERVER} ansible_user=root
+        [db_hosts]
+        benchmark-db ansible_host=${POSTGRES_HOST} ansible_user=root
+        EOL
+
+        mv inventory.ini ansible/playbooks/inventory.ini
diff --git a/.github/workflows/continuous-benchmark-hnsw.yaml b/.github/workflows/continuous-benchmark-hnsw.yaml
@@ -0,0 +1,33 @@
+name: Continuous Benchmark Hnsw Indexing
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run every day at 5am
+    - cron: "0 5 * * *"
+
+# Restrict to only running this workflow one at a time.
+# Any new runs will be queued until the previous run is complete.
+# Any existing pending runs will be cancelled and replaced with current run.
+concurrency:
+  group: continuous-benchmark
+
+jobs:
+  runUpdateHealingBenchmark:
+    runs-on: ubuntu-latest
+    container: alpine/ansible:2.18.1
+    steps:
+      - uses: actions/checkout@v3
+      - uses: webfactory/ssh-agent@v0.8.0
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+      - name: Create inventory
+        uses: ./.github/workflows/actions/create-inventory
+        with:
+          hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
+          db_host: ${{ secrets.POSTGRES_HOST }}
+          server_name: "benchmark-server-3"
+      - name: Run bench
+        id: hnsw-indexing-update
+        run: |
+          cd ansible/playbooks && ansible-playbook playbook-hnsw-index.yml --extra-vars "bench=update"
diff --git a/.gitignore b/.gitignore
@@ -6,3 +6,5 @@ NOTES.md
 
 results/*
 tools/custom/data.json
+
+ansible/playbooks/inventory.ini
diff --git a/ansible/Dockerfile b/ansible/Dockerfile
@@ -0,0 +1,17 @@
+FROM python:3.11-slim
+
+# Set environment variables to prevent prompts during package installation
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+    sshpass \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir ansible
+
+RUN ansible --version
+
+WORKDIR /ansible/playbooks
+
+CMD ["ansible-playbook", "--version"]
diff --git a/ansible/README.md b/ansible/README.md
@@ -0,0 +1,48 @@
+## How to run
+
+### Prerequisites
+* ssh keys (to connect to the remote machines)
+* inventory.ini (to define the actual machine on which the benchmark is run)
+
+Add inventory.ini in [ansible/playbooks/](playbooks) with the following content:
+```ini
+[remote_machines]
+;note that machine's name should be benchmark-machine
+benchmark-machine ansible_host=${YOUR_SERVER_IP} ansible_user=${YOUR_USER}
+[db_hosts]
+benchmark-db ansible_host=${YOUR_SERVER_IP} ansible_user=${YOUR_USER}
+```
+
+### Run ansible inside Docker
+Ensure the ssh keys are properly mounted into the container.
+
+Run the following commands from [ansible](.):
+```bash
+docker buildx build --tag vector-db-benchmark-ansible:latest -f Dockerfile .
+docker run --rm -it -v ~/.ssh/id_rsa:/root/.ssh/id_rsa -v ~/.ssh/id_rsa.pub:/root/.ssh/id_rsa.pub -v ./playbooks:/ansible/playbooks vector-db-benchmark-ansible ansible-playbook playbook-hnsw-index.yml --extra-vars "bench=update"
+```
+
+### Run ansible locally
+The "local" run here means that the ansible command is run locally (so, Ansible should be installed locally).
+The actual machine on which the benchmark is run is defined by the inventory file (see Prerequisites).
+
+Run the following commands from [ansible/playbooks](playbooks):
+```bash
+ansible-playbook playbook-hnsw-index.yml --extra-vars "bench=update"
+```
+
+### Run ansible and benchmark locally
+The "local" run here means that the ansible command is run locally (so, Ansible should be installed locally) and the benchmark is run on the local machine.
+In [ansible/playbooks/](playbooks) add a file `inventory.ini` with the following content:
+```ini
+[remote_machines]
+;note that machine's name should be benchmark-machine
+benchmark-machine ansible_connection=local ansible_user=${YOUR_USER} ansible_become=false
+[db_hosts]
+benchmark-db ansible_host=${YOUR_DB_SERVER_IP} ansible_user=${YOUR_DB_SERVER_USER}
+```
+
+Then from [ansible/playbooks](playbooks) run:
+```bash
+ansible-playbook playbook-hnsw-index.yml --extra-vars "bench=update"
+```
diff --git a/ansible/playbooks/ansible.cfg b/ansible/playbooks/ansible.cfg
@@ -0,0 +1,3 @@
+[defaults]
+host_key_checking = False
+inventory = inventory.ini
diff --git a/ansible/playbooks/files/hnsw-indexing/docker-compose.yaml b/ansible/playbooks/files/hnsw-indexing/docker-compose.yaml
@@ -0,0 +1,21 @@
+version: '3.7'
+
+services:
+  qdrant_bench:
+    image: ${CONTAINER_REGISTRY:-docker.io}/qdrant/qdrant:${QDRANT_VERSION}
+    container_name: qdrant-continuous
+    ports:
+      - "6333:6333"
+      - "6334:6334"
+    logging:
+      driver: "json-file"
+      options:
+        max-file: 1
+        max-size: 10m
+    deploy:
+      resources:
+        limits:
+          memory: 25Gb
+    environment:
+      - QDRANT__LOG_LEVEL=DEBUG
+      - QDRANT__FEATURE_FLAGS__ALL=${QDRANT__FEATURE_FLAGS__ALL:-false}
diff --git a/ansible/playbooks/files/hnsw-indexing/requirements.txt b/ansible/playbooks/files/hnsw-indexing/requirements.txt
@@ -0,0 +1,4 @@
+numpy>=2.2.5
+requests>=2.32.3
+tqdm>=4.67.1
+qdrant-client>=1.14.0
diff --git a/ansible/playbooks/group_vars/hnsw-indexing-update.yml b/ansible/playbooks/group_vars/hnsw-indexing-update.yml
@@ -0,0 +1,17 @@
+qdrant_python_client_version: "1.14.0"
+logging_dir: "/tmp/logs"
+working_dir: "/tmp/experiments"
+dataset_url: "https://storage.googleapis.com/ann-filtered-benchmark/datasets/dbpedia_openai_100K.tgz"
+dataset_name: "dbpedia_openai_100K"
+dataset_dim: "1536"
+servers:
+  - name: "qdrant"
+    registry: "ghcr.io"
+    image: "qdrant/qdrant"
+    version: "dev"
+    feature_flags: "true"
+  - name: "qdrant"
+    registry: "docker.io"
+    image: "qdrant/qdrant"
+    version: "master"
+    feature_flags: "false"
diff --git a/ansible/playbooks/playbook-hnsw-index.yml b/ansible/playbooks/playbook-hnsw-index.yml
@@ -0,0 +1,87 @@
+---
+- name: Run load
+  hosts: remote_machines
+  become: yes
+  tasks:
+    - name: Load common variables
+      include_vars: "group_vars/hnsw-indexing-{{ bench | default('update') }}.yml"
+
+    - name: Ensure necessary packages are installed
+      ansible.builtin.package:
+        name: "{{ item }}"
+        state: present
+      loop:
+        - wget
+        - python3-venv
+
+    - name: Ensure Docker is installed
+      ansible.builtin.command: docker --version
+      register: docker_check
+      failed_when: docker_check.rc not in [0, 127]
+
+    - name: Execute benchmark
+      include_role:
+        name: "run-hnsw-indexing-{{ bench | default('update') }}"
+      loop: "{{ servers }}"
+      loop_control:
+        loop_var: "server"
+      vars:
+        server_name: "{{ server.name }}"
+        server_version: "{{ server.version }}"
+        server_registry: "{{ server.registry }}"
+        server_feature_flags: "{{ server.feature_flags }}"
+        bench: "{{ bench | default('update') }}"
+
+    - name: "Compare results on the remote machine"
+      ansible.builtin.shell: |
+        python3 "${WORK_DIR}/get_score.py"
+      environment:
+        WORK_DIR: "{{ working_dir }}"
+        DATA_DIR: "{{ working_dir }}"
+        BENCH: "{{ bench }}"
+        SERVER_NAME: "{{ servers[0].name }}"
+        SERVER_NAME_2: "{{ servers[1].name }}"
+        SERVER_VERSION: "{{ servers[0].version }}"
+        SERVER_VERSION_2: "{{ servers[1].version }}"
+      register: "score_result"
+
+    - name: Extract precision_score and set fact
+      set_fact:
+        precision_score: "{{ score_result.stdout }}"
+
+- name: Export data into postgres
+  hosts: db_hosts
+  tasks:
+    - name: Load common variables
+      include_vars: "group_vars/hnsw-indexing-{{ bench | default('update') }}.yml"
+
+    - name: Insert data into table
+      ansible.builtin.shell: |
+        result="{{ hostvars['benchmark-machine']['precision_score'] }}"
+        engine="{{ server.name }}-{{ server.version }}"
+        score=$(echo "$result" | grep -oP "${engine}_score=\K[^,]+")
+        indexing_time_s=$(echo "$result" | grep -oP "${engine}_indexing_time=\K[^,]+")
+
+        MEASURE_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+        pg_query="INSERT INTO hnsw_indexing (
+            engine,
+            engine_version,
+            dataset,
+            measure_timestamp,
+            bench_name,
+            precision_score,
+            indexing_time_s
+        ) VALUES (
+            '{{ server.name }}',
+            '{{ server.version }}',
+            '{{ dataset_name }}',
+            '${MEASURE_TIMESTAMP}',
+            '{{ bench }}',
+            '${score}',
+            '${indexing_time_s}'
+        );"
+        docker exec -it qdrant-postgres psql -U qdrant -d postgres -c "${pg_query}"
+      loop: "{{ servers }}"
+      loop_control:
+        loop_var: "server"
diff --git a/ansible/playbooks/roles/run-hnsw-indexing-common/tasks/main.yml b/ansible/playbooks/roles/run-hnsw-indexing-common/tasks/main.yml
@@ -0,0 +1,59 @@
+---
+- name: Copy files to the remote machine
+  ansible.builtin.copy:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: "{{ item.mode | default('0644') }}"
+    owner: "{{ ansible_user }}"
+  loop:
+    - { src: run-bench.sh, dest: "{{ working_dir }}/run-bench.sh", mode: '0755' }
+    - { src: "{{ bench }}.py", dest: "{{ working_dir }}/{{ bench }}.py" }
+    - { src: "../../files/hnsw-indexing/requirements.txt", dest: "{{ working_dir }}/requirements.txt" }
+    - { src: "../../files/hnsw-indexing/docker-compose.yaml", dest: "{{ working_dir }}/docker-compose.yaml" }
+    - { src: get_score.py, dest: "{{ working_dir }}/get_score.py" }
+
+- name: Start Docker container on the remote machine
+  ansible.builtin.shell: |
+    docker compose down
+    pkill qdrant
+    docker rm -f qdrant-continuous || true
+    docker rmi -f ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION} || true
+    docker compose up -d
+  args:
+    chdir: "{{ working_dir }}"
+  environment:
+    QDRANT_VERSION: "{{ server_version }}"
+    CONTAINER_REGISTRY: "{{ server_registry }}"
+    QDRANT__FEATURE_FLAGS__ALL: "{{ server_feature_flags }}"
+  register: compose_result
+  failed_when: compose_result.rc != 0
+
+- name: "Execute the script on the remote machine: {{ server_name }}-{{ server_version }}"
+  ansible.builtin.shell: |
+    {{ working_dir }}/run-bench.sh > "{{ working_dir }}/log-{{ server_name }}-{{ server_version }}-{{ bench }}.log" 2>&1
+  async: 3600 # 60 minutes
+  poll: 30 # Check every 30 seconds
+  environment:
+    OUTPUT_FILENAME: "{{ working_dir }}/output-{{ server_name }}-{{ server_version }}-{{ bench }}.json"
+    BENCH: "{{ bench }}"
+    WORK_DIR: "{{ working_dir }}"
+    DATASET_NAME: "{{ dataset_name }}"
+    DATASET_DIM: "{{ dataset_dim }}"
+    DATASET_NAME_2: "{{ dataset_2_name | default('') }}"
+  register: bench_result
+  ignore_errors: yes
+
+- name: Show script output if failed
+  ansible.builtin.shell: cat "{{ working_dir }}/log-{{ server_name }}-{{ server_version }}-{{ bench }}.log"
+  when: bench_result.rc != 0
+  register: script_output
+
+- name: Print script output
+  ansible.builtin.debug:
+    msg: "{{ script_output.stdout }}"
+  when: bench_result.rc != 0
+
+- name: Fail if script failed
+  ansible.builtin.fail:
+    msg: "Script failed with rc={{ bench_result.rc }}"
+  when: bench_result.rc != 0
diff --git a/ansible/playbooks/roles/run-hnsw-indexing-update/files/get_score.py b/ansible/playbooks/roles/run-hnsw-indexing-update/files/get_score.py
@@ -0,0 +1,43 @@
+import json
+import os.path
+
+SERVER_NAME = os.getenv("SERVER_NAME", "qdrant")
+SERVER_NAME_2 = os.getenv("SERVER_NAME_2", "qdrant")
+SERVER_VERSION = os.getenv("SERVER_VERSION", "dev")
+SERVER_VERSION_2 = os.getenv("SERVER_VERSION_2", "master")
+BENCH = os.getenv("BENCH", "update")
+DATA_DIR = os.getenv("DATA_DIR", "data")
+
+filepaths = {
+    f"{SERVER_NAME}-{SERVER_VERSION}": os.path.join(
+        DATA_DIR, f"output-{SERVER_NAME}-{SERVER_VERSION}-{BENCH}.json"
+    ),
+    f"{SERVER_NAME_2}-{SERVER_VERSION_2}": os.path.join(
+        DATA_DIR, f"output-{SERVER_NAME_2}-{SERVER_VERSION_2}-{BENCH}.json"
+    ),
+}
+
+
+def main():
+    results = {}
+
+    for label, path in filepaths.items():
+        with open(path, "r") as file:
+            text = file.read()
+            given_output = json.loads(text)
+
+        precision_before_iteration = given_output.get("precision_before_iteration", 0.0)
+        precision_after_iteration = given_output.get("precision_after_iteration", 1.0)
+        score = round(precision_before_iteration / precision_after_iteration, 4)
+        indexing_time = given_output.get("indexing_total_time_s", 0.0)
+        results[label] = {"score": score, "indexing_time": indexing_time}
+
+    result = ""
+    for label, data in results.items():
+        result += f"{label}_score={data['score']},{label}_indexing_time={data['indexing_time']},"
+    print(result)
+    return result
+
+
+if __name__ == "__main__":
+    main()
diff --git a/ansible/playbooks/roles/run-hnsw-indexing-update/files/run-bench.sh b/ansible/playbooks/roles/run-hnsw-indexing-update/files/run-bench.sh
diff --git a/ansible/playbooks/roles/run-hnsw-indexing-update/files/update.py b/ansible/playbooks/roles/run-hnsw-indexing-update/files/update.py
diff --git a/ansible/playbooks/roles/run-hnsw-indexing-update/tasks/main.yml b/ansible/playbooks/roles/run-hnsw-indexing-update/tasks/main.yml

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[defaults]`
	`2`	`+host_key_checking = False`
	`3`	`+inventory = inventory.ini`