fix: Correct the image_url for calling gpt vision (#1004)

Author: cyberflying

code/backend/batch/utilities/helpers/embedders/push_embedder.py

@@ -103,7 +103,7 @@ def __generate_image_caption(self, source_url):
                         "text": "Describe this image in detail. Limit the response to 500 words.",
                         "type": "text",
                     },
-                    {"image_url": source_url, "type": "image_url"},
+                    {"image_url": {"url": source_url}, "type": "image_url"},
                 ],
             },
         ]

code/backend/batch/utilities/tools/question_answer_tool.py

@@ -137,7 +137,7 @@ def generate_on_your_data_messages(
                         [
                             {
                                 "type": "image_url",
-                                "image_url": image_url,
+                                "image_url": {"url": image_url},
                             }
                             for image_url in image_urls
                         ]

code/tests/functional/tests/backend_api/default/test_advanced_image_processing.py

@@ -266,7 +266,7 @@ def test_image_urls_included_in_call_to_openai(
                                 "type": "text",
                                 "text": '## Retrieved Documents\n{"retrieved_documents":[{"[doc1]":{"content":"content"}}]}\n\n## User Question\nWhat is the meaning of life?',
                             },
-                            {"type": "image_url", "image_url": ANY},
+                            {"type": "image_url", "image_url": {"url": ANY}},
                         ],
                         "role": "user",
                     },
@@ -286,6 +286,6 @@ def test_image_urls_included_in_call_to_openai(
         ),
     )[0]
 
-    assert request.json["messages"][6]["content"][1]["image_url"].startswith(
+    assert request.json["messages"][6]["content"][1]["image_url"]["url"].startswith(
         "https://source"
     )

code/tests/functional/tests/functions/advanced_image_processing/test_advanced_image_processing.py

@@ -174,7 +174,7 @@ def test_image_passed_to_llm_to_generate_caption(
                                 "text": "Describe this image in detail. Limit the response to 500 words.",
                                 "type": "text",
                             },
-                            {"image_url": ANY, "type": "image_url"},
+                            {"image_url": {"url": ANY}, "type": "image_url"},
                         ],
                     },
                 ],
@@ -192,7 +192,7 @@ def test_image_passed_to_llm_to_generate_caption(
         ),
     )[0]
 
-    assert request.get_json()["messages"][1]["content"][1]["image_url"].startswith(
+    assert request.get_json()["messages"][1]["content"][1]["image_url"]["url"].startswith(
         f"{app_config.get('AZURE_STORAGE_ACCOUNT_ENDPOINT')}{app_config.get('AZURE_BLOB_CONTAINER_NAME')}/{FILE_NAME}"
     )
 

code/tests/utilities/helpers/test_push_embedder.py

@@ -164,7 +164,7 @@ def test_embed_file_advanced_image_processing_uses_vision_model_for_captioning(
                         "text": "Describe this image in detail. Limit the response to 500 words.",
                         "type": "text",
                     },
-                    {"image_url": source_url, "type": "image_url"},
+                    {"image_url": {"url": source_url}, "type": "image_url"},
                 ],
             },
         ],

code/tests/utilities/tools/test_question_answer_tool.py

@@ -339,7 +339,7 @@ def test_use_advanced_vision_processing(env_helper_mock, llm_helper_mock):
                         "type": "text",
                         "text": 'Sources: {"retrieved_documents":[{"[doc1]":{"content":"mock content"}},{"[doc2]":{"content":"mock content 2"}}]}, Question: mock question',
                     },
-                    {"type": "image_url", "image_url": "mock source 2mock sas"},
+                    {"type": "image_url", "image_url": {"url": "mock source 2mock sas"}},
                 ],
                 "role": "user",
             },
@@ -411,7 +411,7 @@ def test_limit_number_of_images_passed_to_llm(
                         "type": "text",
                         "text": 'Sources: {"retrieved_documents":[{"[doc1]":{"content":"mock content"}},{"[doc2]":{"content":"mock content 2"}},{"[doc3]":{"content":"mock content 3"}}]}, Question: mock question',
                     },
-                    {"type": "image_url", "image_url": "mock source 2mock sas"},
+                    {"type": "image_url", "image_url": {"url": "mock source 2mock sas"}},
                 ],
                 "role": "user",
             },