Test vision feature, refactor vector_fields to make sense, address feedback

Author: pamelafox

.azdo/pipelines/azure-dev.yml

@@ -61,7 +61,6 @@ steps:
       AZURE_SEARCH_SEMANTIC_RANKER: $(AZURE_SEARCH_SEMANTIC_RANKER)
       AZURE_SEARCH_QUERY_REWRITING: $(AZURE_SEARCH_QUERY_REWRITING)
       AZURE_SEARCH_FIELD_NAME_EMBEDDING: $(AZURE_SEARCH_FIELD_NAME_EMBEDDING)
-      AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING: $(AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING)
       AZURE_STORAGE_ACCOUNT: $(AZURE_STORAGE_ACCOUNT)
       AZURE_STORAGE_RESOURCE_GROUP: $(AZURE_STORAGE_RESOURCE_GROUP)
       AZURE_STORAGE_SKU: $(AZURE_STORAGE_SKU)

.github/workflows/azure-dev.yml

@@ -51,7 +51,6 @@ jobs:
       AZURE_SEARCH_SEMANTIC_RANKER: ${{ vars.AZURE_SEARCH_SEMANTIC_RANKER }}
       AZURE_SEARCH_QUERY_REWRITING: ${{ vars.AZURE_SEARCH_QUERY_REWRITING }}
       AZURE_SEARCH_FIELD_NAME_EMBEDDING: ${{ vars.AZURE_SEARCH_FIELD_NAME_EMBEDDING }}
-      AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING: ${{ vars.AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING }}
       AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }}
       AZURE_STORAGE_RESOURCE_GROUP: ${{ vars.AZURE_STORAGE_RESOURCE_GROUP }}
       AZURE_STORAGE_SKU: ${{ vars.AZURE_STORAGE_SKU }}

app/backend/app.py

@@ -465,7 +465,6 @@ async def setup_clients():
     AZURE_SEARCH_QUERY_REWRITING = os.getenv("AZURE_SEARCH_QUERY_REWRITING", "false").lower()
     # This defaults to the previous field name "embedding", for backwards compatibility
     AZURE_SEARCH_FIELD_NAME_EMBEDDING = os.getenv("AZURE_SEARCH_FIELD_NAME_EMBEDDING", "embedding")
-    AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING = os.getenv("AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING", "imageEmbedding")
 
     AZURE_SPEECH_SERVICE_ID = os.getenv("AZURE_SPEECH_SERVICE_ID")
     AZURE_SPEECH_SERVICE_LOCATION = os.getenv("AZURE_SPEECH_SERVICE_LOCATION")
@@ -586,7 +585,6 @@ async def setup_clients():
             embeddings=text_embeddings_service,
             file_processors=file_processors,
             search_field_name_embedding=AZURE_SEARCH_FIELD_NAME_EMBEDDING,
-            search_field_name_image_embedding=AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING,
         )
         current_app.config[CONFIG_INGESTER] = ingester
 

app/backend/approaches/approach.py

@@ -41,8 +41,6 @@
 class Document:
     id: Optional[str]
     content: Optional[str]
-    embedding: Optional[List[float]]
-    image_embedding: Optional[List[float]]
     category: Optional[str]
     sourcepage: Optional[str]
     sourcefile: Optional[str]
@@ -56,9 +54,6 @@ def serialize_for_results(self) -> dict[str, Any]:
         result_dict = {
             "id": self.id,
             "content": self.content,
-            # Should we rename to its actual field name in the index?
-            "embedding": Document.trim_embedding(self.embedding),
-            "imageEmbedding": Document.trim_embedding(self.image_embedding),
             "category": self.category,
             "sourcepage": self.sourcepage,
             "sourcefile": self.sourcefile,
@@ -81,18 +76,6 @@ def serialize_for_results(self) -> dict[str, Any]:
         }
         return result_dict
 
-    @classmethod
-    def trim_embedding(cls, embedding: Optional[List[float]]) -> Optional[str]:
-        """Returns a trimmed list of floats from the vector embedding."""
-        if embedding:
-            if len(embedding) > 2:
-                # Format the embedding list to show the first 2 items followed by the count of the remaining items."""
-                return f"[{embedding[0]}, {embedding[1]} ...+{len(embedding) - 2} more]"
-            else:
-                return str(embedding)
-
-        return None
-
 
 @dataclass
 class ThoughtStep:
@@ -245,8 +228,6 @@ async def search(
                     Document(
                         id=document.get("id"),
                         content=document.get("content"),
-                        embedding=document.get(self.embedding_field),
-                        image_embedding=document.get("imageEmbedding"),
                         category=document.get("category"),
                         sourcepage=document.get("sourcepage"),
                         sourcefile=document.get("sourcefile"),
@@ -321,13 +302,14 @@ class ExtraArgs(TypedDict, total=False):
             **dimensions_args,
         )
         query_vector = embedding.data[0].embedding
-        # TODO: use optimizations from rag time journey 3
+        # This performs an oversampling due to how the search index was setup,
+        # so we do not need to explicitly pass in an oversampling parameter here
         return VectorizedQuery(vector=query_vector, k_nearest_neighbors=50, fields=self.embedding_field)
 
     async def compute_image_embedding(self, q: str):
         endpoint = urljoin(self.vision_endpoint, "computervision/retrieval:vectorizeText")
         headers = {"Content-Type": "application/json"}
-        params = {"api-version": "2023-02-01-preview", "modelVersion": "latest"}
+        params = {"api-version": "2024-02-01", "model-version": "2023-04-15"}
         data = {"text": q}
 
         headers["Authorization"] = "Bearer " + await self.vision_token_provider()

app/backend/approaches/chatreadretrievereadvision.py

@@ -90,7 +90,7 @@ async def run_until_final_call(
         minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
         filter = self.build_filter(overrides, auth_claims)
 
-        vector_fields = overrides.get("vector_fields", [self.embedding_field])
+        vector_fields = overrides.get("vector_fields", "textAndImageEmbeddings")
         send_text_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "texts", None]
         send_images_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "images", None]
 
@@ -123,13 +123,10 @@ async def run_until_final_call(
         # If retrieval mode includes vectors, compute an embedding for the query
         vectors = []
         if use_vector_search:
-            for field in vector_fields:
-                vector = (
-                    await self.compute_image_embedding(query_text)
-                    if field.startswith("image")
-                    else await self.compute_text_embedding(query_text)
-                )
-                vectors.append(vector)
+            if vector_fields == "textEmbeddingOnly" or vector_fields == "textAndImageEmbeddings":
+                vectors.append(await self.compute_text_embedding(query_text))
+            if vector_fields == "imageEmbeddingOnly" or vector_fields == "textAndImageEmbeddings":
+                vectors.append(await self.compute_image_embedding(query_text))
 
         results = await self.search(
             top,

app/backend/approaches/retrievethenreadvision.py

@@ -85,20 +85,17 @@ async def run(
         minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0)
         filter = self.build_filter(overrides, auth_claims)
 
-        vector_fields = overrides.get("vector_fields", [self.embedding_field])
+        vector_fields = overrides.get("vector_fields", "textAndImageEmbeddings")
         send_text_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "texts", None]
         send_images_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "images", None]
 
         # If retrieval mode includes vectors, compute an embedding for the query
         vectors = []
         if use_vector_search:
-            for field in vector_fields:
-                vector = (
-                    await self.compute_image_embedding(q)
-                    if field.startswith("image")
-                    else await self.compute_text_embedding(q)
-                )
-                vectors.append(vector)
+            if vector_fields == "textEmbeddingOnly" or vector_fields == "textAndImageEmbeddings":
+                vectors.append(await self.compute_text_embedding(q))
+            if vector_fields == "imageEmbeddingOnly" or vector_fields == "textAndImageEmbeddings":
+                vectors.append(await self.compute_image_embedding(q))
 
         results = await self.search(
             top,

app/backend/prepdocs.py

@@ -399,7 +399,6 @@ async def main(strategy: Strategy, setup_index: bool = True):
             document_action=document_action,
             embeddings=openai_embeddings_service,
             search_field_name_embedding=os.environ["AZURE_SEARCH_FIELD_NAME_EMBEDDING"],
-            search_field_name_image_embedding=os.environ["AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING"],
             subscription_id=os.environ["AZURE_SUBSCRIPTION_ID"],
             search_service_user_assigned_id=args.searchserviceassignedid,
             search_analyzer_name=os.getenv("AZURE_SEARCH_ANALYZER_NAME"),
@@ -434,7 +433,6 @@ async def main(strategy: Strategy, setup_index: bool = True):
             search_analyzer_name=os.getenv("AZURE_SEARCH_ANALYZER_NAME"),
             # Default to the previous field names for backward compatibility
             search_field_name_embedding=os.getenv("AZURE_SEARCH_FIELD_NAME_EMBEDDING", "embedding"),
-            search_field_name_image_embedding=os.getenv("AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING", "imageEmbedding"),
             use_acls=use_acls,
             category=args.category,
             use_content_understanding=use_content_understanding,

app/backend/prepdocslib/embeddings.py

@@ -238,7 +238,7 @@ def __init__(self, endpoint: str, token_provider: Callable[[], Awaitable[str]]):
     async def create_embeddings(self, blob_urls: List[str]) -> List[List[float]]:
         endpoint = urljoin(self.endpoint, "computervision/retrieval:vectorizeImage")
         headers = {"Content-Type": "application/json"}
-        params = {"api-version": "2023-02-01-preview", "modelVersion": "latest"}
+        params = {"api-version": "2024-02-01", "model-version": "2023-04-15"}
         headers["Authorization"] = "Bearer " + await self.token_provider()
 
         embeddings: List[List[float]] = []

app/backend/prepdocslib/filestrategy.py

@@ -52,7 +52,6 @@ def __init__(
         image_embeddings: Optional[ImageEmbeddings] = None,
         search_analyzer_name: Optional[str] = None,
         search_field_name_embedding: Optional[str] = None,
-        search_field_name_image_embedding: Optional[str] = None,
         use_acls: bool = False,
         category: Optional[str] = None,
         use_content_understanding: bool = False,
@@ -66,7 +65,6 @@ def __init__(
         self.image_embeddings = image_embeddings
         self.search_analyzer_name = search_analyzer_name
         self.search_field_name_embedding = search_field_name_embedding
-        self.search_field_name_image_embedding = search_field_name_image_embedding
         self.search_info = search_info
         self.use_acls = use_acls
         self.category = category
@@ -81,7 +79,6 @@ def setup_search_manager(self):
             False,
             self.embeddings,
             field_name_embedding=self.search_field_name_embedding,
-            field_name_image_embedding=self.search_field_name_image_embedding,
             search_images=self.image_embeddings is not None,
         )
 
@@ -137,7 +134,6 @@ def __init__(
         embeddings: Optional[OpenAIEmbeddings] = None,
         image_embeddings: Optional[ImageEmbeddings] = None,
         search_field_name_embedding: Optional[str] = None,
-        search_field_name_image_embedding: Optional[str] = None,
     ):
         self.file_processors = file_processors
         self.embeddings = embeddings
@@ -150,11 +146,9 @@ def __init__(
             use_int_vectorization=False,
             embeddings=self.embeddings,
             field_name_embedding=search_field_name_embedding,
-            field_name_image_embedding=search_field_name_image_embedding,
             search_images=False,
         )
         self.search_field_name_embedding = search_field_name_embedding
-        self.search_field_name_image_embedding = search_field_name_image_embedding
 
     async def add_file(self, file: File):
         if self.image_embeddings:

app/backend/prepdocslib/integratedvectorizerstrategy.py

@@ -41,7 +41,6 @@ def __init__(
         search_info: SearchInfo,
         embeddings: AzureOpenAIEmbeddingService,
         search_field_name_embedding: str,
-        search_field_name_image_embedding: str,
         subscription_id: str,
         search_service_user_assigned_id: str,
         document_action: DocumentAction = DocumentAction.Add,
@@ -55,7 +54,6 @@ def __init__(
         self.document_action = document_action
         self.embeddings = embeddings
         self.search_field_name_embedding = search_field_name_embedding
-        self.search_field_name_image_embedding = search_field_name_image_embedding
         self.subscription_id = subscription_id
         self.search_user_assigned_identity = search_service_user_assigned_id
         self.search_analyzer_name = search_analyzer_name
@@ -139,7 +137,6 @@ async def setup(self):
             use_int_vectorization=True,
             embeddings=self.embeddings,
             field_name_embedding=self.search_field_name_embedding,
-            field_name_image_embedding=self.search_field_name_image_embedding,
             search_images=False,
         )