Merge pull request #219 from qdrant/feat/keep-datasets

feat/keep-datasets

Author: generall

.github/workflows/clean-datasets.yaml

@@ -0,0 +1,52 @@
+name: Clean Datasets
+
+on:
+  repository_dispatch:
+  workflow_dispatch:
+  schedule:
+    # Run every month on the 1st day at 3 am
+    - cron: "0 3 1 * *"
+
+concurrency:
+  group: continuous-benchmark
+
+# This removes the ci-datasets volume from client machine.
+# The next run of Continuous Benchmark will create the volume again and download all the datasets.
+jobs:
+  removeDatasetsVolume:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: webfactory/[email protected]
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+      - name: Benches
+        id: benches
+        run: |
+            export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
+
+            set +e
+
+            timeout 10m bash -x tools/run_client_remove_volume.sh
+
+            set -e
+      - name: Send Notification
+        if: failure()
+        uses: slackapi/[email protected]
+        with:
+          payload: |
+            {
+              "text": "Failed to remove the datasets volume (removeDatasetsVolume), run status: ${{ job.status }}",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "View the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

tools/run_client_remove_volume.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+PS4='ts=$(date "+%Y-%m-%dT%H:%M:%SZ") level=DEBUG line=$LINENO file=$BASH_SOURCE '
+set -euo pipefail
+
+CLOUD_NAME=${CLOUD_NAME:-"hetzner"}
+SERVER_USERNAME=${SERVER_USERNAME:-"root"}
+
+SCRIPT=$(realpath "$0")
+SCRIPT_PATH=$(dirname "$SCRIPT")
+
+BENCH_CLIENT_NAME=${CLIENT_NAME:-"benchmark-client-1"}
+
+IP_OF_THE_CLIENT=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_public_ip.sh" "$BENCH_CLIENT_NAME")
+
+echo "Remove ci-datasets volume from client"
+RUN_CMD="docker volume rm -f ci-datasets || true"
+
+ssh -tt -o ServerAliveInterval=120 -o ServerAliveCountMax=10 "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "${RUN_CMD}"

tools/run_client_script.sh

@@ -39,6 +39,7 @@ if [[ "$EXPERIMENT_MODE" == "snapshot" ]]; then
 
 else
   scp "${SCRIPT_PATH}/run_experiment.sh" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/run_experiment.sh"
+  scp "${SCRIPT_PATH}/../datasets/datasets.json" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/datasets.json"
 
   RUN_EXPERIMENT="ENGINE_NAME=${ENGINE_NAME} \
   DATASETS=${DATASETS} \

tools/run_experiment.sh

@@ -45,13 +45,23 @@ if [[ "$EXPERIMENT_MODE" != "snapshot" ]]; then
   docker rmi --force qdrant/vector-db-benchmark:latest || true
 fi
 
+echo "Ensure datasets volume exists and contains latest datasets.json"
+docker volume create ci-datasets
+if [[ -f "$HOME/datasets.json" ]]; then
+  echo "Found datasets.json, update the volume"
+  mv ~/datasets.json "$(docker volume inspect ci-datasets -f '{{ .Mountpoint }}')"
+else
+  echo "datasets.json is missing, do not update the volume"
+fi
+
 if [[ "$EXPERIMENT_MODE" == "full" ]] || [[ "$EXPERIMENT_MODE" == "upload" ]]; then
   echo "EXPERIMENT_MODE=$EXPERIMENT_MODE"
   docker run \
     --rm \
     -it \
     --name ci-benchmark-upload \
     -v "$HOME/results:/code/results" \
+    -v "ci-datasets:/code/datasets" \
     qdrant/vector-db-benchmark:latest \
     python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-search
 fi
@@ -70,6 +80,7 @@ if [[ "$EXPERIMENT_MODE" == "full" ]] || [[ "$EXPERIMENT_MODE" == "search" ]]; t
     -it \
     --name ci-benchmark-search \
     -v "$HOME/results:/code/results" \
+    -v "ci-datasets:/code/datasets" \
     qdrant/vector-db-benchmark:latest \
     python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-upload
 fi
@@ -85,6 +96,7 @@ if [[ "$EXPERIMENT_MODE" == "parallel" ]]; then
     --rm \
     --name ci-benchmark-upload \
     -v "$HOME/results/parallel:/code/results" \
+    -v "ci-datasets:/code/datasets" \
     qdrant/vector-db-benchmark:latest \
     python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-search --skip-configure &
   UPLOAD_PID=$!
@@ -94,6 +106,7 @@ if [[ "$EXPERIMENT_MODE" == "parallel" ]]; then
     --rm \
     --name ci-benchmark-search \
     -v "$HOME/results/parallel:/code/results" \
+    -v "ci-datasets:/code/datasets" \
     qdrant/vector-db-benchmark:latest \
     python run.py --engines "${ENGINE_NAME}" --datasets "${DATASETS}" --host "${PRIVATE_IP_OF_THE_SERVER}" --no-skip-if-exists --skip-upload &
   SEARCH_PID=$!