Pre-build and commit search indexes

Author: eepMoody

.github/workflows/rebuild_search_index.yml

@@ -0,0 +1,75 @@
+name: "Rebuild Search Index"
+
+# Runs when data files change on staging
+# Rebuilds indexes, commits them, then triggers the build workflow
+on:
+  push:
+    branches:
+      - staging
+    paths:
+      - 'data/**'
+
+jobs:
+  rebuild-index:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install pipenv
+        run: pip install pipenv
+
+      - name: Cache pipenv virtualenv
+        uses: actions/cache@v4
+        with:
+          path: ~/.local/share/virtualenvs
+          key: ${{ runner.os }}-pipenv-${{ hashFiles('Pipfile.lock') }}
+
+      - name: Install dependencies
+        run: pipenv install --deploy
+
+      - name: Download spaCy model
+        run: pipenv run python -m spacy download en_core_web_md
+
+      - name: Check if indexes are stale
+        id: check
+        run: |
+          if pipenv run python manage.py indexctl check; then
+            echo "stale=false" >> $GITHUB_OUTPUT
+          else
+            echo "stale=true" >> $GITHUB_OUTPUT
+          fi
+        env:
+          SECRET_KEY: "github-actions-build-key"
+
+      - name: Rebuild and pack indexes
+        if: steps.check.outputs.stale == 'true'
+        run: |
+          echo "Indexes are stale, rebuilding..."
+          pipenv run python manage.py quicksetup --rebuild-index
+          pipenv run python manage.py indexctl pack
+        env:
+          SECRET_KEY: "github-actions-build-key"
+
+      - name: Commit updated indexes
+        if: steps.check.outputs.stale == 'true'
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add search/indexes/ search/index_manifest.json
+          if git diff --staged --quiet; then
+            echo "No changes to commit"
+          else
+            git commit -m "Rebuild search indexes
+
+            Triggered by data changes in ${{ github.sha }}"
+            git push
+          fi
+

.gitignore

@@ -19,9 +19,10 @@ db.sqlite3-journal
 # Django static files
 staticfiles/
 
-# Search indexes
+# Search indexes (unpacked - these are generated from search/indexes/)
 server/whoosh_index/
 whoosh_index/
+server/vector_index.pkl
 
 # IDE and editor configurations
 .vscode/
@@ -33,7 +34,5 @@ api/tests/approved_files/*.recieved.*
 # Generated files
 openapi-schema.yml
 
-server/vector_index.pkl
-
 # Temporary files for magic items update
 temp_magic_items/

README.md

@@ -74,24 +74,31 @@ pipenv install --dev
 
 ## Build
 
-Crate a local database and import game content. 
+Create a local database, import game content, and set up search indexes:
+
 ```bash
-pipenv run python manage.py quicksetup --noindex
+pipenv run python manage.py quicksetup
 ```
 
-To make sure the API is always using your updated code, this command must be run again if:
-- You add/remove/edit Game Content
-- You edit Python code
-- You switch git branches
+Run this again if you switch git branches or pull new changes.
 
 
 ### Search Indexing
 
-To use the search function, you must build the search index by running the above command without the `--noindex` flag.
+Search indexes are pre-built and included in the repo. Running `quicksetup` unpacks them automatically:
+
 ```bash
 pipenv run python manage.py quicksetup
 ```
 
+If you've changed data in `data/`, rebuild the indexes before committing:
+
+```bash
+pipenv run python manage.py quickindex
+```
+
+This takes 2-3 minutes and updates `search/indexes/` which should be committed with your data changes.
+
 ## Run
 
 Run the server locally. This server is only for development and shall __not__ be used in production. The server will be available at `http://localhost:8000`.

api/management/commands/quicksetup.py

@@ -17,7 +17,12 @@ def add_arguments(self, parser: argparse.ArgumentParser):
         parser.add_argument(
             "--noindex",
             action="store_true",
-            help="Skip building search indexes.",
+            help="Skip unpacking/building search indexes.",
+        )
+        parser.add_argument(
+            "--rebuild-index",
+            action="store_true",
+            help="Force rebuild indexes from data (slow, ~2-3 min).",
         )
         parser.add_argument(
             "--clean",
@@ -67,12 +72,14 @@ def handle(self, *args, **options):
                 ))
                 return
 
-            if not options['noindex']:
-                if settings.BUILD_V2_INDEX:
-                    self.stdout.write('Building the search index...')
-                    build_search_index()
-            else:
-                self.stdout.write('Skipping index build because of --noindex.')
+            if options['noindex']:
+                self.stdout.write('Skipping indexes because of --noindex.')
+            elif options['rebuild_index']:
+                self.stdout.write('Rebuilding search indexes from data (this takes 2-3 min)...')
+                build_search_index()
+            elif settings.BUILD_V2_INDEX:
+                self.stdout.write('Setting up search indexes...')
+                setup_search_index()
 
         self.stdout.write(self.style.SUCCESS('API setup complete.'))
 
@@ -121,3 +128,18 @@ def collect_static():
 
 def build_search_index():
     call_command('buildindex', '--v1', '--v2')
+
+
+def setup_search_index():
+    """Unpack pre-built indexes if available, otherwise build from scratch."""
+    archive_dir = Path("search/indexes")
+    whoosh_archive = archive_dir / "whoosh_index.tar.gz"
+    vector_archive = archive_dir / "vector_index.pkl.gz"
+    
+    if whoosh_archive.exists() and vector_archive.exists():
+        print("Unpacking pre-built indexes...")
+        call_command('indexctl', 'unpack')
+    else:
+        print("No pre-built indexes found, building from scratch...")
+        print("(This takes 2-3 min. Run 'manage.py indexctl pack' to avoid this next time.)")
+        build_search_index()

search/index_manifest.json

@@ -0,0 +1,4 @@
+{
+  "data_hash": "1d4de910e94fdbc038b2168426aea7ad",
+  "version": 1
+}

search/indexes/vector_index.pkl.gz

@@ -8,16 +8,16 @@
 from django.core.management import call_command
 from django.core.management.base import BaseCommand
 
-from django.db import connection, transaction
+from django.db import connection
 
 from api import models as v1
 from api_v2 import models as v2
 from search import models as search
 
 class Command(BaseCommand):
-    """Implementation for the `manage.py `index_v1` subcommand."""
+    """Build search indexes for text, fuzzy, and semantic search."""
 
-    help = 'Build the v1 search index.'
+    help = 'Build the search indexes (FTS, Whoosh, and vector).'
 
     def add_arguments(self, parser: argparse.ArgumentParser):
         """Define arguments for the `manage.py quicksetup` subcommand."""
@@ -194,7 +194,7 @@ def build_vector_index(self):
         print("Saving vector index to disk...")
 
         index_data = {
-            "names": all_names,
+            "names": all_names, 
             "metadata": all_metadata,
             "embeddings": embeddings,
             "vector_size": nlp.vocab.vectors_length