update for vector_norm map

Author: rbs333

redisvl/extensions/llmcache/semantic.py

@@ -22,7 +22,7 @@
     SemanticCacheIndexSchema,
 )
 from redisvl.index import AsyncSearchIndex, SearchIndex
-from redisvl.query import RangeQuery
+from redisvl.query import VectorRangeQuery
 from redisvl.query.filter import FilterExpression
 from redisvl.query.query import BaseQuery
 from redisvl.redis.connection import RedisConnectionFactory
@@ -390,7 +390,7 @@ def check(
         vector = vector or self._vectorize_prompt(prompt)
         self._check_vector_dims(vector)
 
-        query = RangeQuery(
+        query = VectorRangeQuery(
             vector=vector,
             vector_field_name=CACHE_VECTOR_FIELD_NAME,
             return_fields=self.return_fields,
@@ -473,14 +473,15 @@ async def acheck(
         vector = vector or await self._avectorize_prompt(prompt)
         self._check_vector_dims(vector)
 
-        query = RangeQuery(
+        query = VectorRangeQuery(
             vector=vector,
             vector_field_name=CACHE_VECTOR_FIELD_NAME,
             return_fields=self.return_fields,
             distance_threshold=distance_threshold,
             num_results=num_results,
             return_score=True,
             filter_expression=filter_expression,
+            normalize_vector_distance=True,
         )
 
         # Search the cache!

redisvl/extensions/router/schema.py

@@ -18,7 +18,7 @@ class Route(BaseModel):
     """List of reference phrases for the route."""
     metadata: Dict[str, Any] = Field(default={})
     """Metadata associated with the route."""
-    distance_threshold: Annotated[float, Field(strict=True, gt=0, le=1)] = 0.5
+    distance_threshold: Annotated[float, Field(strict=True, gt=0, le=2)] = 0.5
     """Distance threshold for matching the route."""
 
     @field_validator("name")

redisvl/extensions/router/semantic.py

@@ -17,7 +17,7 @@
     SemanticRouterIndexSchema,
 )
 from redisvl.index import SearchIndex
-from redisvl.query import RangeQuery
+from redisvl.query import VectorRangeQuery
 from redisvl.redis.utils import convert_bytes, hashify, make_dict
 from redisvl.utils.log import get_logger
 from redisvl.utils.utils import deprecated_argument, model_to_dict
@@ -237,7 +237,7 @@ def _distance_threshold_filter(self) -> str:
 
     def _build_aggregate_request(
         self,
-        vector_range_query: RangeQuery,
+        vector_range_query: VectorRangeQuery,
         aggregation_method: DistanceAggregationMethod,
         max_k: int,
     ) -> AggregateRequest:
@@ -279,7 +279,7 @@ def _get_route_matches(
         # therefore you might take the max_threshold and further refine from there.
         distance_threshold = max(route.distance_threshold for route in self.routes)
 
-        vector_range_query = RangeQuery(
+        vector_range_query = VectorRangeQuery(
             vector=vector,
             vector_field_name=ROUTE_VECTOR_FIELD_NAME,
             distance_threshold=float(distance_threshold),

redisvl/index/index.py

@@ -49,7 +49,7 @@
 )
 from redisvl.redis.utils import convert_bytes
 from redisvl.schema import IndexSchema, StorageType
-from redisvl.schema.fields import VectorDistanceMetric
+from redisvl.schema.fields import VECTOR_NORM_MAP, VectorDistanceMetric
 from redisvl.utils.log import get_logger
 
 logger = get_logger(__name__)
@@ -92,12 +92,20 @@ def process_results(
         and not query._return_fields  # type: ignore
     )
 
-    normalize_cosine_distance = (
-        (isinstance(query, VectorQuery) or isinstance(query, VectorRangeQuery))
-        and query._normalize_cosine_distance
-        and schema.fields[query._vector_field_name].attrs.distance_metric  # type: ignore
-        == VectorDistanceMetric.COSINE
-    )
+    if (
+        isinstance(query, VectorQuery) or isinstance(query, VectorRangeQuery)
+    ) and query._normalize_vector_distance:
+        dist_metric = VectorDistanceMetric(
+            schema.fields[query._vector_field_name].attrs.distance_metric.upper()  # type: ignore
+        )
+        if dist_metric == VectorDistanceMetric.IP:
+            warnings.warn(
+                "Attempting to normalize inner product distance metric. Use cosine distance instead which is normalized inner product by definition."
+            )
+
+        norm_fn = VECTOR_NORM_MAP[dist_metric.value]
+    else:
+        norm_fn = None
 
     # Process records
     def _process(doc: "Document") -> Dict[str, Any]:
@@ -112,10 +120,10 @@ def _process(doc: "Document") -> Dict[str, Any]:
                 return {"id": doc_dict.get("id"), **json_data}
             raise ValueError(f"Unable to parse json data from Redis {json_data}")
 
-        if normalize_cosine_distance:
+        if norm_fn:
             # convert float back to string to be consistent
             doc_dict[query.DISTANCE_ID] = str(  # type: ignore
-                norm_cosine_distance(float(doc_dict[query.DISTANCE_ID]))  # type: ignore
+                norm_fn(float(doc_dict[query.DISTANCE_ID]))  # type: ignore
             )
 
         # Remove 'payload' if present

redisvl/query/query.py

@@ -198,7 +198,7 @@ def __init__(
         in_order: bool = False,
         hybrid_policy: Optional[str] = None,
         batch_size: Optional[int] = None,
-        normalize_cosine_distance: bool = False,
+        normalize_vector_distance: bool = False,
     ):
         """A query for running a vector search along with an optional filter
         expression.
@@ -234,9 +234,12 @@ def __init__(
                 of vectors to fetch in each batch. Larger values may improve performance
                 at the cost of memory usage. Only applies when hybrid_policy="BATCHES".
                 Defaults to None, which lets Redis auto-select an appropriate batch size.
-            normalize_cosine_distance (bool): by default Redis returns cosine distance as a value
-                between 0 and 2 where 0 is the best match. If set to True, the cosine distance will be
-                converted to cosine similarity with a value between 0 and 1 where 1 is the best match.
+            normalize_vector_distance (bool): Redis supports 3 distance metrics: L2 (euclidean),
+                IP (inner product), and COSINE. By default, L2 distance returns an unbounded value.
+                COSINE distance returns a value between 0 and 2. IP returns a value determined by
+                the magnitude of the vector. Setting this flag to true converts COSINE and L2 distance
+                to a similarity score between 0 and 1. Note: setting this flag to true for IP will
+                throw a warning since by definition COSINE similarity is normalized IP.
 
         Raises:
             TypeError: If filter_expression is not of type redisvl.query.FilterExpression
@@ -250,7 +253,7 @@ def __init__(
         self._num_results = num_results
         self._hybrid_policy: Optional[HybridPolicy] = None
         self._batch_size: Optional[int] = None
-        self._normalize_cosine_distance = normalize_cosine_distance
+        self._normalize_vector_distance = normalize_vector_distance
         self.set_filter(filter_expression)
         query_string = self._build_query_string()
 
@@ -399,7 +402,7 @@ def __init__(
         in_order: bool = False,
         hybrid_policy: Optional[str] = None,
         batch_size: Optional[int] = None,
-        normalize_cosine_distance: bool = False,
+        normalize_vector_distance: bool = False,
     ):
         """A query for running a filtered vector search based on semantic
         distance threshold.
@@ -443,9 +446,12 @@ def __init__(
                 of vectors to fetch in each batch. Larger values may improve performance
                 at the cost of memory usage. Only applies when hybrid_policy="BATCHES".
                 Defaults to None, which lets Redis auto-select an appropriate batch size.
-            normalize_cosine_distance (bool): by default Redis returns cosine distance as a value
-                between 0 and 2 where 0 is the best match. If set to True, the cosine distance will be
-                converted to cosine similarity with a value between 0 and 1 where 1 is the best match.
+            normalize_vector_distance (bool): Redis supports 3 distance metrics: L2 (euclidean),
+                IP (inner product), and COSINE. By default, L2 distance returns an unbounded value.
+                COSINE distance returns a value between 0 and 2. IP returns a value determined by
+                the magnitude of the vector. Setting this flag to true converts COSINE and L2 distance
+                to a similarity score between 0 and 1. Note: setting this flag to true for IP will
+                throw a warning since by definition COSINE similarity is normalized IP.
 
         Raises:
             TypeError: If filter_expression is not of type redisvl.query.FilterExpression
@@ -472,7 +478,7 @@ def __init__(
         if batch_size is not None:
             self.set_batch_size(batch_size)
 
-        self._normalize_cosine_distance = normalize_cosine_distance
+        self._normalize_vector_distance = normalize_vector_distance
         self.set_distance_threshold(distance_threshold)
         self.set_filter(filter_expression)
         query_string = self._build_query_string()

redisvl/schema/fields.py

@@ -16,6 +16,14 @@
 from redis.commands.search.field import TextField as RedisTextField
 from redis.commands.search.field import VectorField as RedisVectorField
 
+from redisvl.utils.utils import norm_cosine_distance, norm_l2_distance
+
+VECTOR_NORM_MAP = {
+    "COSINE": norm_cosine_distance,
+    "L2": norm_l2_distance,
+    "IP": None,  # normalized inner product is cosine similarity by definition
+}
+
 
 class FieldTypes(str, Enum):
     TAG = "tag"

redisvl/utils/utils.py

@@ -198,3 +198,10 @@ def norm_cosine_distance(value: float) -> float:
     Normalize the cosine distance to a similarity score between 0 and 1.
     """
     return (2 - value) / 2
+
+
+def norm_l2_distance(value: float) -> float:
+    """
+    Normalize the L2 distance.
+    """
+    return 1 / (1 + value)

tests/conftest.py

@@ -141,7 +141,7 @@ def sample_data(sample_datetimes):
             "last_updated": sample_datetimes["high"].timestamp(),
             "credit_score": "medium",
             "location": "-110.0839,37.3861",
-            "user_embedding": [0.9, 0.9, 0.1],
+            "user_embedding": [-0.1, -0.1, -0.5],
         },
     ]
 

tests/integration/test_query.py

@@ -25,6 +25,7 @@ def vector_query():
     return VectorQuery(
         vector=[0.1, 0.1, 0.5],
         vector_field_name="user_embedding",
+        return_score=True,
         return_fields=[
             "user",
             "credit_score",
@@ -58,7 +59,7 @@ def normalized_vector_query():
     return VectorQuery(
         vector=[0.1, 0.1, 0.5],
         vector_field_name="user_embedding",
-        normalize_cosine_distance=True,
+        normalize_vector_distance=True,
         return_score=True,
         return_fields=[
             "user",
@@ -107,7 +108,7 @@ def normalized_range_query():
     return RangeQuery(
         vector=[0.1, 0.1, 0.5],
         vector_field_name="user_embedding",
-        normalize_cosine_distance=True,
+        normalize_vector_distance=True,
         return_score=True,
         return_fields=["user", "credit_score", "age", "job", "location"],
         distance_threshold=0.2,
@@ -749,13 +750,28 @@ def test_query_normalize_cosine_distance(index, normalized_vector_query):
         assert 0 <= float(r["vector_distance"]) <= 1
 
 
-def test_query_normalize_cosine_distance_lp_distance(L2_index, normalized_vector_query):
+def test_query_cosine_distance_un_normalized(index, vector_query):
 
-    res = L2_index.query(normalized_vector_query)
+    res = index.query(vector_query)
+
+    assert any(float(r["vector_distance"]) > 1 for r in res)
+
+
+def test_query_l2_distance_un_normalized(L2_index, vector_query):
+
+    res = L2_index.query(vector_query)
 
     assert any(float(r["vector_distance"]) > 1 for r in res)
 
 
+def test_query_l2_distance_normalized(L2_index, normalized_vector_query):
+
+    res = L2_index.query(normalized_vector_query)
+
+    for r in res:
+        assert 0 <= float(r["vector_distance"]) <= 1
+
+
 def test_range_query_normalize_cosine_distance(index, normalized_range_query):
 
     res = index.query(normalized_range_query)