update tests, docs, and formatting/linting

Author: tylerhutcherson

docs/user_guide/01_getting_started.ipynb

@@ -163,7 +163,7 @@ def from_yaml(cls, schema_path: str, **kwargs):
 
             from redisvl.index import SearchIndex
 
-            index = SearchIndex.from_yaml("schemas/schema.yaml")
+            index = SearchIndex.from_yaml("schemas/schema.yaml", redis_url="redis://localhost:6379")
         """
         schema = IndexSchema.from_yaml(schema_path)
         return cls(schema=schema, **kwargs)
@@ -191,7 +191,7 @@ def from_dict(cls, schema_dict: Dict[str, Any], **kwargs):
                 "fields": [
                     {"name": "doc-id", "type": "tag"}
                 ]
-            })
+            }, redis_url="redis://localhost:6379")
 
         """
         schema = IndexSchema.from_dict(schema_dict)
@@ -235,10 +235,14 @@ class SearchIndex(BaseSearchIndex):
         from redisvl.index import SearchIndex
 
         # initialize the index object with schema from file
-        index = SearchIndex.from_yaml("schemas/schema.yaml", redis_url="redis://localhost:6379")
+        index = SearchIndex.from_yaml(
+            "schemas/schema.yaml",
+            redis_url="redis://localhost:6379",
+            validate_on_load=True
+        )
 
         # create the index
-        index.create(overwrite=True)
+        index.create(overwrite=True, drop=False)
 
         # data is an iterable of dictionaries
         index.load(data)
@@ -387,11 +391,6 @@ def connect(self, redis_url: Optional[str] = None, **kwargs):
             ValueError: If the Redis URL is not provided nor accessible
                 through the `REDIS_URL` environment variable.
             ModuleNotFoundError: If required Redis modules are not installed.
-
-        .. code-block:: python
-
-            index.connect(redis_url="redis://localhost:6379")
-
         """
         self.__redis_client = RedisConnectionFactory.get_redis_connection(
             redis_url=redis_url, **kwargs
@@ -411,16 +410,6 @@ def set_client(self, redis_client: redis.Redis, **kwargs):
 
         Raises:
             TypeError: If the provided client is not valid.
-
-        .. code-block:: python
-
-            import redis
-            from redisvl.index import SearchIndex
-
-            client = redis.Redis.from_url("redis://localhost:6379")
-            index = SearchIndex.from_yaml("schemas/schema.yaml")
-            index.set_client(client)
-
         """
         RedisConnectionFactory.validate_sync_redis(redis_client)
         self.__redis_client = redis_client
@@ -799,11 +788,12 @@ class AsyncSearchIndex(BaseSearchIndex):
         # initialize the index object with schema from file
         index = AsyncSearchIndex.from_yaml(
             "schemas/schema.yaml",
-            redis_url="redis://localhost:6379"
+            redis_url="redis://localhost:6379",
+            validate_on_load=True
         )
 
         # create the index
-        await index.create(overwrite=True)
+        await index.create(overwrite=True, drop=False)
 
         # data is an iterable of dictionaries
         await index.load(data)

docs/user_guide/data_validation.ipynb

@@ -143,7 +143,7 @@ async def _aget(client: AsyncRedis, key: str) -> Dict[str, Any]:
         """
         raise NotImplementedError
 
-    def validate(self, obj: Dict[str, Any]) -> Dict[str, Any]:
+    def _validate(self, obj: Dict[str, Any]) -> Dict[str, Any]:
         """
         Validate an object against the schema using Pydantic-based validation.
 
@@ -161,7 +161,7 @@ def validate(self, obj: Dict[str, Any]) -> Dict[str, Any]:
 
     def _preprocess_and_validate_objects(
         self,
-        objects: List[Any],
+        objects: Iterable[Any],
         id_field: Optional[str] = None,
         keys: Optional[Iterable[str]] = None,
         preprocess: Optional[Callable] = None,
@@ -201,7 +201,7 @@ def _preprocess_and_validate_objects(
 
                 # Schema validation if enabled
                 if validate:
-                    processed_obj = self.validate(processed_obj)
+                    processed_obj = self._validate(processed_obj)
 
                 # Store valid object with its key for writing
                 prepared_objects.append((key, processed_obj))
@@ -263,7 +263,7 @@ def write(
 
         # Pass 1: Preprocess and validate all objects
         prepared_objects = self._preprocess_and_validate_objects(
-            objects,
+            list(objects),  # Convert Iterable to List
             id_field=id_field,
             keys=keys,
             preprocess=preprocess,
@@ -342,7 +342,7 @@ async def awrite(
 
         # Pass 1: Preprocess and validate all objects
         prepared_objects = self._preprocess_and_validate_objects(
-            objects,
+            list(objects),  # Convert Iterable to List
             id_field=id_field,
             keys=keys,
             preprocess=preprocess,

redisvl/index/index.py

@@ -8,7 +8,7 @@
 import json
 import re
 import warnings
-from typing import Any, Dict, List, Optional, Type, Union
+from typing import Any, Dict, List, Optional, Type, Union, cast
 
 from pydantic import BaseModel, Field, field_validator
 
@@ -53,7 +53,7 @@ def get_model_for_schema(cls, schema: IndexSchema) -> Type[BaseModel]:
     @classmethod
     def _map_field_to_pydantic_type(
         cls, field: BaseField, storage_type: StorageType
-    ) -> Type:
+    ) -> Type[Any]:
         """
         Map Redis field types to appropriate Pydantic types.
 
@@ -72,14 +72,17 @@ def _map_field_to_pydantic_type(
         elif field.type == FieldTypes.TAG:
             return str
         elif field.type == FieldTypes.NUMERIC:
-            return Union[int, float]
+            return Union[int, float]  # type: ignore
         elif field.type == FieldTypes.GEO:
             return str
         elif field.type == FieldTypes.VECTOR:
             # For JSON storage, vectors are always lists
             if storage_type == StorageType.JSON:
                 # For int data types, vectors must be ints, otherwise floats
-                if field.attrs.datatype in (VectorDataType.INT8, VectorDataType.UINT8):
+                if field.attrs.datatype in (  # type: ignore
+                    VectorDataType.INT8,
+                    VectorDataType.UINT8,
+                ):
                     return List[int]
                 return List[float]
             else:
@@ -103,8 +106,8 @@ def _create_model(cls, schema: IndexSchema) -> Type[BaseModel]:
         storage_type = schema.index.storage_type
 
         # Create annotations dictionary for the dynamic model
-        annotations = {}
-        class_dict = {}
+        annotations: Dict[str, Any] = {}
+        class_dict: Dict[str, Any] = {}
 
         # Build annotations and field metadata
         for field_name, field in schema.fields.items():
@@ -154,6 +157,8 @@ def _disallow_bool(cls, value):
 
             # Register validators for VECTOR fields
             elif field.type == FieldTypes.VECTOR:
+                dims = field.attrs.dims  # type: ignore
+                datatype = field.attrs.datatype  # type: ignore
 
                 def make_vector_validator(
                     fname: str, dims: int, datatype: VectorDataType
@@ -190,7 +195,7 @@ def _validate_vector(cls, value):
                     return _validate_vector
 
                 class_dict[f"validate_{field_name}"] = make_vector_validator(
-                    field_name, field.attrs.dims, field.attrs.datatype
+                    field_name, dims, datatype
                 )
 
         # Create class dictionary with annotations and field metadata

redisvl/index/storage.py

@@ -4,7 +4,7 @@
 from redis import Redis as SyncRedis
 from redis.asyncio import Redis
 
-from redisvl.exceptions import RedisSearchError
+from redisvl.exceptions import RedisSearchError, RedisVLError
 from redisvl.index import AsyncSearchIndex
 from redisvl.query import VectorQuery
 from redisvl.redis.utils import convert_bytes
@@ -267,7 +267,7 @@ async def test_search_index_load_preprocess(async_index):
     await async_index.create(overwrite=True, drop=True)
     data = [{"id": "1", "test": "foo"}]
 
-    async def preprocess(record):
+    def preprocess(record):
         record["test"] = "bar"
         return record
 
@@ -279,10 +279,10 @@ async def preprocess(record):
         == "bar"
     )
 
-    async def bad_preprocess(record):
+    def bad_preprocess(record):
         return 1
 
-    with pytest.raises(ValueError):
+    with pytest.raises(RedisVLError):
         await async_index.load(data, id_field="id", preprocess=bad_preprocess)
 
 
@@ -298,7 +298,7 @@ async def test_no_id_field(async_index):
     bad_data = [{"wrong_key": "1", "value": "test"}]
 
     # catch missing / invalid id_field
-    with pytest.raises(ValueError):
+    with pytest.raises(RedisVLError):
         await async_index.load(bad_data, id_field="key")
 
 

redisvl/schema/validation.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from redisvl.exceptions import RedisSearchError
+from redisvl.exceptions import RedisSearchError, RedisVLError
 from redisvl.index import SearchIndex
 from redisvl.query import VectorQuery
 from redisvl.redis.utils import convert_bytes
@@ -265,7 +265,7 @@ def preprocess(record):
     def bad_preprocess(record):
         return 1
 
-    with pytest.raises(ValueError):
+    with pytest.raises(RedisVLError):
         index.load(data, id_field="id", preprocess=bad_preprocess)
 
 
@@ -274,7 +274,7 @@ def test_no_id_field(index):
     bad_data = [{"wrong_key": "1", "value": "test"}]
 
     # catch missing / invalid id_field
-    with pytest.raises(ValueError):
+    with pytest.raises(RedisVLError):
         index.load(bad_data, id_field="key")