Replace Elasticsearch with Whoosh and spaCy for in-process search

Use Whoosh for text/fuzzy search and spaCy word embeddings for
semantic similarity. Removes the need for a separate ES service.

Author: eepMoody

.do/app.yaml

@@ -19,8 +19,9 @@ db.sqlite3-journal
 # Django static files
 staticfiles/
 
-# Search index
+# Search indexes
 server/whoosh_index/
+whoosh_index/
 
 # IDE and editor configurations
 .vscode/

.gitignore

@@ -12,12 +12,14 @@ COPY . /opt/services/open5e-api
 # Install all dependencies from Pipfile
 RUN pipenv install -v
 
+# Download spaCy model for semantic search
+RUN pipenv run python -m spacy download en_core_web_md
+
 # Remove .env file (set your env vars via docker-compose.yml or your hosting provider)
 RUN rm -f .env
 
-# Copy startup script
-COPY start.sh /start.sh
-RUN chmod +x /start.sh
+# Run setup
+RUN pipenv run python manage.py quicksetup
 
-# Run startup script (handles migrations, quicksetup, and gunicorn)
-CMD ["/start.sh"]
+# Run gunicorn
+CMD ["pipenv", "run", "gunicorn", "-b", ":8080", "--workers", "2", "--timeout", "120", "server.wsgi:application"]

Dockerfile

@@ -13,10 +13,10 @@ requests = "*"
 whitenoise = "*"
 gunicorn = "*"
 drf-spectacular = {extras = ["sidecar"], version = "*"}
-elasticsearch = ">=7.0.0,<8.0.0"
 django-haystack = "*"
+Whoosh = "*"
 numpy = "<2.0"
-scikit-learn = "*"
+spacy = ">=3.0"
 
 [dev-packages]
 pytest = "*"

Pipfile

@@ -14,15 +14,11 @@ class Command(BaseCommand):
     """Implementation for the `manage.py quicksetup` subcommand."""
 
     def add_arguments(self, parser: argparse.ArgumentParser):
-        """Define arguments for the `manage.py quicksetup` subcommand."""
-
-        # Named (optional) arguments.
         parser.add_argument(
             "--noindex",
             action="store_true",
-            help="Flushes all existing database data before adding new objects.",
+            help="Skip building search indexes.",
         )
-
         parser.add_argument(
             "--clean",
             action="store_true",
@@ -56,15 +52,7 @@ def handle(self, *args, **options):
                 self.stdout.write(self.style.ERROR(
                     'QUICKSETUP FAILED - Fix foreign key constraint violations before proceeding.'
                 ))
-                return  # Exit without showing "API setup complete"
-            
-            if not options['noindex']:
-                if settings.BUILD_V1_INDEX:
-                    build_haystack_index()
-            else:
-                self.stdout.write("Skipping v1 index build because of --noindex")
-        else:
-            self.stdout.write('Skipping v1 database population.')
+                return
 
         if settings.INCLUDE_V2_DATA:
             self.stdout.write('Populating the v2 database...')
@@ -77,73 +65,59 @@ def handle(self, *args, **options):
                 self.stdout.write(self.style.ERROR(
                     'QUICKSETUP FAILED - Fix foreign key constraint violations before proceeding.'
                 ))
-                return  # Exit without showing "API setup complete"
+                return
 
             if not options['noindex']:
                 if settings.BUILD_V2_INDEX:
-                    self.stdout.write('Building the v2 index with both v1 and v2 data.')
-                    build_v1v2_searchindex()
+                    self.stdout.write('Building the search index...')
+                    build_search_index()
             else:
-                self.stdout.write('Skipping v2 index build because of --noindex.')
+                self.stdout.write('Skipping index build because of --noindex.')
 
         self.stdout.write(self.style.SUCCESS('API setup complete.'))
 
 
-def migrate_db() -> None:
-    """Migrate the local database as needed to incorporate new model updates.
-    This command is added primarily to assist in local development, because
-    checking out and changing branches results in unclean model/dbs."""
-
+def migrate_db():
     call_command('makemigrations')
     call_command('migrate')
 
-def is_dirty() ->None:
-    # TODO switch these over to server settings values.
-    is_dirty=False
-    if Path('./server/whoosh_index').is_dir():
+
+def is_dirty():
+    is_dirty = False
+    if Path('whoosh_index').is_dir():
         print("Found whoosh_index")
-        is_dirty=True
+        is_dirty = True
     if Path(settings.STATIC_ROOT).is_dir():
         print("Found static root")
-        is_dirty=True
+        is_dirty = True
     if Path(settings.DATABASES['default']['NAME']).exists():
         print("Found db file")
-        is_dirty=True
+        is_dirty = True
     return is_dirty
 
-def clean_dir() ->None:
-    if Path('./server/whoosh_index').is_dir():
-        shutil.rmtree(Path('./server/whoosh_index'))
+
+def clean_dir():
+    if Path('whoosh_index').is_dir():
+        shutil.rmtree(Path('whoosh_index'))
     if Path(settings.STATIC_ROOT).is_dir():
         shutil.rmtree(Path(settings.STATIC_ROOT))
     if Path(settings.DATABASES['default']['NAME']).exists():
         Path(settings.DATABASES['default']['NAME']).unlink()
-    vector_index = Path('server/vector_index.pkl')
-    if vector_index.exists():
-        vector_index.unlink()
+    if Path('server/vector_index.pkl').exists():
+        Path('server/vector_index.pkl').unlink()
 
-def import_v1() -> None:
-    """Import the v1 apps' database models."""
+
+def import_v1():
     call_command('import', '--dir', 'data/v1')
 
 
-def import_v2() -> None:
-    """Import the v2 apps' database models."""
+def import_v2():
     call_command('import', '--dir', 'data/v2')
 
 
-def collect_static() -> None:
-    """Collect static files in a single location."""
+def collect_static():
     call_command('collectstatic', '--noinput')
 
 
-def build_haystack_index() -> None:
-    """Freshen the haystack search indexes. This is an internal haystack
-    API that is being called, and only applies to v1 data."""
-    print("THIS ENTIRE COMMAND HAS BEEN DEPRECATED! EXPECT ERRORS.")
-    call_command('update_index', '--remove')
-
-def build_v1v2_searchindex() -> None:
-    """Builds the custom search index defined in the api_v2 management
-    commands. Only adds the v1 data."""
-    call_command('buildindex','--v1','--v2')
+def build_search_index():
+    call_command('buildindex', '--v1', '--v2')

Pipfile.lock

@@ -1,42 +1,15 @@
 services:
-  elasticsearch:
-    image: elasticsearch:8.11.0
-    container_name: elasticsearch
-    environment:
-      - discovery.type=single-node
-      - xpack.security.enabled=false
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-      - http.max_content_length=100mb
-      - indices.memory.index_buffer_size=20%
-    ports:
-      - "9200:9200"
-    volumes:
-      - elasticsearch_data:/usr/share/elasticsearch/data
-    healthcheck:
-      test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=60s && curl -f http://localhost:9200/_cat/nodes?format=json | grep -q '\"name\"' || exit 1"]
-      interval: 15s
-      timeout: 30s
-      retries: 10
-      start_period: 60s
-
   web:
     build: .
     container_name: open5e-api
     ports:
       - "8080:8080"
     environment:
       - SECRET_KEY=${SECRET_KEY:-your-secret-key-here}
-      - ELASTICSEARCH_URL=http://elasticsearch:9200/
-    depends_on:
-      elasticsearch:
-        condition: service_healthy
     healthcheck:
       test: ["CMD-SHELL", "curl -f http://localhost:8080/v2/spells/?limit=1 || exit 1"]
       interval: 30s
       timeout: 10s
       retries: 5
     mem_limit: 1g
     mem_reservation: 512m
-
-volumes:
-  elasticsearch_data: 

api/management/commands/quicksetup.py

@@ -15,16 +15,6 @@ class SearchConfig(AppConfig):
 
     def ready(self):
         """Initialize search components when Django starts."""
-        try:
-            logger.info("Initializing search components...")
-            from .services import get_tfidf_vectorizer
-            
-            # Only initialize the vectorizer factory, don't build index during startup
-            # This prevents hanging when Elasticsearch is not available during Django startup
-            logger.info("TF-IDF vectorizer factory initialized")
-            
-            # Vector index will be built lazily on first search request
-            logger.info("Search components initialized successfully")
-        except Exception as e:
-            logger.error(f"Error initializing search components: {e}")
-            # Don't crash the app if search initialization fails
+        # Search components are loaded lazily on first use
+        # No initialization needed during Django startup
+        logger.info("Search app ready - components will load on first use")