diff --git a/.github/workflows/python-test.yaml b/.github/workflows/python-test.yaml index d242dc1ec5..9234f54f26 100644 --- a/.github/workflows/python-test.yaml +++ b/.github/workflows/python-test.yaml @@ -36,6 +36,7 @@ jobs: version: "0.4.20" cache-dependency-glob: "requirements**.txt" python-version: ${{ matrix.python_version }} + activate-environment: true - name: Setup node uses: actions/setup-node@v4 with: diff --git a/app/backend/app.py b/app/backend/app.py index 9a8c5c9864..1b4563bb98 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -23,6 +23,7 @@ get_bearer_token_provider, ) from azure.monitor.opentelemetry import configure_azure_monitor +from azure.search.documents.agent.aio import KnowledgeAgentRetrievalClient from azure.search.documents.aio import SearchClient from azure.search.documents.indexes.aio import SearchIndexClient from azure.storage.blob.aio import ContainerClient @@ -57,6 +58,8 @@ from approaches.retrievethenreadvision import RetrieveThenReadVisionApproach from chat_history.cosmosdb import chat_history_cosmosdb_bp from config import ( + CONFIG_AGENT_CLIENT, + CONFIG_AGENTIC_RETRIEVAL_ENABLED, CONFIG_ASK_APPROACH, CONFIG_ASK_VISION_APPROACH, CONFIG_AUTH_CLIENT, @@ -308,6 +311,7 @@ def config(): "showSpeechOutputAzure": current_app.config[CONFIG_SPEECH_OUTPUT_AZURE_ENABLED], "showChatHistoryBrowser": current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED], "showChatHistoryCosmos": current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED], + "showAgenticRetrievalOption": current_app.config[CONFIG_AGENTIC_RETRIEVAL_ENABLED], } ) @@ -424,10 +428,14 @@ async def setup_clients(): AZURE_USERSTORAGE_ACCOUNT = os.environ.get("AZURE_USERSTORAGE_ACCOUNT") AZURE_USERSTORAGE_CONTAINER = os.environ.get("AZURE_USERSTORAGE_CONTAINER") AZURE_SEARCH_SERVICE = os.environ["AZURE_SEARCH_SERVICE"] + AZURE_SEARCH_ENDPOINT = f"https://{AZURE_SEARCH_SERVICE}.search.windows.net" AZURE_SEARCH_INDEX = os.environ["AZURE_SEARCH_INDEX"] + AZURE_SEARCH_AGENT = os.getenv("AZURE_SEARCH_AGENT", "") # Shared by all OpenAI deployments OPENAI_HOST = os.getenv("OPENAI_HOST", "azure") OPENAI_CHATGPT_MODEL = os.environ["AZURE_OPENAI_CHATGPT_MODEL"] + AZURE_OPENAI_SEARCHAGENT_MODEL = os.getenv("AZURE_OPENAI_SEARCHAGENT_MODEL") + AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT = os.getenv("AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT") OPENAI_EMB_MODEL = os.getenv("AZURE_OPENAI_EMB_MODEL_NAME", "text-embedding-ada-002") OPENAI_EMB_DIMENSIONS = int(os.getenv("AZURE_OPENAI_EMB_DIMENSIONS") or 1536) OPENAI_REASONING_EFFORT = os.getenv("AZURE_OPENAI_REASONING_EFFORT") @@ -479,6 +487,7 @@ async def setup_clients(): USE_SPEECH_OUTPUT_AZURE = os.getenv("USE_SPEECH_OUTPUT_AZURE", "").lower() == "true" USE_CHAT_HISTORY_BROWSER = os.getenv("USE_CHAT_HISTORY_BROWSER", "").lower() == "true" USE_CHAT_HISTORY_COSMOS = os.getenv("USE_CHAT_HISTORY_COSMOS", "").lower() == "true" + USE_AGENTIC_RETRIEVAL = os.getenv("USE_AGENTIC_RETRIEVAL", "").lower() == "true" # WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep RUNNING_ON_AZURE = os.getenv("WEBSITE_HOSTNAME") is not None or os.getenv("RUNNING_IN_PRODUCTION") is not None @@ -513,10 +522,13 @@ async def setup_clients(): # Set up clients for AI Search and Storage search_client = SearchClient( - endpoint=f"https://{AZURE_SEARCH_SERVICE}.search.windows.net", + endpoint=AZURE_SEARCH_ENDPOINT, index_name=AZURE_SEARCH_INDEX, credential=azure_credential, ) + agent_client = KnowledgeAgentRetrievalClient( + endpoint=AZURE_SEARCH_ENDPOINT, agent_name=AZURE_SEARCH_AGENT, credential=azure_credential + ) blob_container_client = ContainerClient( f"https://{AZURE_STORAGE_ACCOUNT}.blob.core.windows.net", AZURE_STORAGE_CONTAINER, credential=azure_credential @@ -527,7 +539,7 @@ async def setup_clients(): if AZURE_USE_AUTHENTICATION: current_app.logger.info("AZURE_USE_AUTHENTICATION is true, setting up search index client") search_index_client = SearchIndexClient( - endpoint=f"https://{AZURE_SEARCH_SERVICE}.search.windows.net", + endpoint=AZURE_SEARCH_ENDPOINT, credential=azure_credential, ) search_index = await search_index_client.get_index(AZURE_SEARCH_INDEX) @@ -645,6 +657,7 @@ async def setup_clients(): current_app.config[CONFIG_OPENAI_CLIENT] = openai_client current_app.config[CONFIG_SEARCH_CLIENT] = search_client + current_app.config[CONFIG_AGENT_CLIENT] = agent_client current_app.config[CONFIG_BLOB_CONTAINER_CLIENT] = blob_container_client current_app.config[CONFIG_AUTH_CLIENT] = auth_helper @@ -668,6 +681,7 @@ async def setup_clients(): current_app.config[CONFIG_SPEECH_OUTPUT_AZURE_ENABLED] = USE_SPEECH_OUTPUT_AZURE current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED] = USE_CHAT_HISTORY_BROWSER current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED] = USE_CHAT_HISTORY_COSMOS + current_app.config[CONFIG_AGENTIC_RETRIEVAL_ENABLED] = USE_AGENTIC_RETRIEVAL prompt_manager = PromptyManager() @@ -675,6 +689,10 @@ async def setup_clients(): # RetrieveThenReadApproach is used by /ask for single-turn Q&A current_app.config[CONFIG_ASK_APPROACH] = RetrieveThenReadApproach( search_client=search_client, + search_index_name=AZURE_SEARCH_INDEX, + agent_model=AZURE_OPENAI_SEARCHAGENT_MODEL, + agent_deployment=AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT, + agent_client=agent_client, openai_client=openai_client, auth_helper=auth_helper, chatgpt_model=OPENAI_CHATGPT_MODEL, @@ -694,6 +712,10 @@ async def setup_clients(): # ChatReadRetrieveReadApproach is used by /chat for multi-turn conversation current_app.config[CONFIG_CHAT_APPROACH] = ChatReadRetrieveReadApproach( search_client=search_client, + search_index_name=AZURE_SEARCH_INDEX, + agent_model=AZURE_OPENAI_SEARCHAGENT_MODEL, + agent_deployment=AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT, + agent_client=agent_client, openai_client=openai_client, auth_helper=auth_helper, chatgpt_model=OPENAI_CHATGPT_MODEL, diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index 734e606690..bc2f918860 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -2,17 +2,20 @@ from abc import ABC from collections.abc import AsyncGenerator, Awaitable from dataclasses import dataclass -from typing import ( - Any, - Callable, - Optional, - TypedDict, - Union, - cast, -) +from typing import Any, Callable, Optional, TypedDict, Union, cast from urllib.parse import urljoin import aiohttp +from azure.search.documents.agent.aio import KnowledgeAgentRetrievalClient +from azure.search.documents.agent.models import ( + KnowledgeAgentAzureSearchDocReference, + KnowledgeAgentIndexParams, + KnowledgeAgentMessage, + KnowledgeAgentMessageTextContent, + KnowledgeAgentRetrievalRequest, + KnowledgeAgentRetrievalResponse, + KnowledgeAgentSearchActivityRecord, +) from azure.search.documents.aio import SearchClient from azure.search.documents.models import ( QueryCaptionResult, @@ -36,16 +39,17 @@ @dataclass class Document: - id: Optional[str] - content: Optional[str] - category: Optional[str] - sourcepage: Optional[str] - sourcefile: Optional[str] - oids: Optional[list[str]] - groups: Optional[list[str]] - captions: list[QueryCaptionResult] + id: Optional[str] = None + content: Optional[str] = None + category: Optional[str] = None + sourcepage: Optional[str] = None + sourcefile: Optional[str] = None + oids: Optional[list[str]] = None + groups: Optional[list[str]] = None + captions: Optional[list[QueryCaptionResult]] = None score: Optional[float] = None reranker_score: Optional[float] = None + search_agent_query: Optional[str] = None def serialize_for_results(self) -> dict[str, Any]: result_dict = { @@ -70,6 +74,7 @@ def serialize_for_results(self) -> dict[str, Any]: ), "score": self.score, "reranker_score": self.reranker_score, + "search_agent_query": self.search_agent_query, } return result_dict @@ -247,6 +252,67 @@ async def search( return qualified_documents + async def run_agentic_retrieval( + self, + messages: list[ChatCompletionMessageParam], + agent_client: KnowledgeAgentRetrievalClient, + search_index_name: str, + top: Optional[int] = None, + filter_add_on: Optional[str] = None, + minimum_reranker_score: Optional[float] = None, + max_docs_for_reranker: Optional[int] = None, + ) -> tuple[KnowledgeAgentRetrievalResponse, list[Document]]: + # STEP 1: Invoke agentic retrieval + response = await agent_client.retrieve( + retrieval_request=KnowledgeAgentRetrievalRequest( + messages=[ + KnowledgeAgentMessage( + role=str(msg["role"]), content=[KnowledgeAgentMessageTextContent(text=str(msg["content"]))] + ) + for msg in messages + if msg["role"] != "system" + ], + target_index_params=[ + KnowledgeAgentIndexParams( + index_name=search_index_name, + reranker_threshold=minimum_reranker_score, + max_docs_for_reranker=max_docs_for_reranker, + filter_add_on=filter_add_on, + include_reference_source_data=True, + ) + ], + ) + ) + + # STEP 2: Generate a contextual and content specific answer using the search results and chat history + activities = response.activity + activity_mapping = ( + { + activity.id: activity.query.search if activity.query else "" + for activity in activities + if isinstance(activity, KnowledgeAgentSearchActivityRecord) + } + if activities + else {} + ) + + results = [] + if response and response.references: + for reference in response.references: + if isinstance(reference, KnowledgeAgentAzureSearchDocReference) and reference.source_data: + results.append( + Document( + id=reference.doc_key, + content=reference.source_data["content"], + sourcepage=reference.source_data["sourcepage"], + search_agent_query=activity_mapping[reference.activity_source], + ) + ) + if top and len(results) == top: + break + + return response, results + def get_sources_content( self, results: list[Document], use_semantic_captions: bool, use_image_citation: bool ) -> list[str]: diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index d795d6573e..772529ec7d 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -1,6 +1,7 @@ from collections.abc import Awaitable from typing import Any, Optional, Union, cast +from azure.search.documents.agent.aio import KnowledgeAgentRetrievalClient from azure.search.documents.aio import SearchClient from azure.search.documents.models import VectorQuery from openai import AsyncOpenAI, AsyncStream @@ -28,6 +29,10 @@ def __init__( self, *, search_client: SearchClient, + search_index_name: str, + agent_model: Optional[str], + agent_deployment: Optional[str], + agent_client: KnowledgeAgentRetrievalClient, auth_helper: AuthenticationHelper, openai_client: AsyncOpenAI, chatgpt_model: str, @@ -44,6 +49,10 @@ def __init__( reasoning_effort: Optional[str] = None, ): self.search_client = search_client + self.search_index_name = search_index_name + self.agent_model = agent_model + self.agent_deployment = agent_deployment + self.agent_client = agent_client self.openai_client = openai_client self.auth_helper = auth_helper self.chatgpt_model = chatgpt_model @@ -70,6 +79,56 @@ async def run_until_final_call( auth_claims: dict[str, Any], should_stream: bool = False, ) -> tuple[ExtraInfo, Union[Awaitable[ChatCompletion], Awaitable[AsyncStream[ChatCompletionChunk]]]]: + use_agentic_retrieval = True if overrides.get("use_agentic_retrieval") else False + original_user_query = messages[-1]["content"] + + reasoning_model_support = self.GPT_REASONING_MODELS.get(self.chatgpt_model) + if reasoning_model_support and (not reasoning_model_support.streaming and should_stream): + raise Exception( + f"{self.chatgpt_model} does not support streaming. Please use a different model or disable streaming." + ) + if use_agentic_retrieval: + extra_info = await self.run_agentic_retrieval_approach(messages, overrides, auth_claims) + else: + extra_info = await self.run_search_approach(messages, overrides, auth_claims) + + messages = self.prompt_manager.render_prompt( + self.answer_prompt, + self.get_system_prompt_variables(overrides.get("prompt_template")) + | { + "include_follow_up_questions": bool(overrides.get("suggest_followup_questions")), + "past_messages": messages[:-1], + "user_query": original_user_query, + "text_sources": extra_info.data_points.text, + }, + ) + + chat_coroutine = cast( + Union[Awaitable[ChatCompletion], Awaitable[AsyncStream[ChatCompletionChunk]]], + self.create_chat_completion( + self.chatgpt_deployment, + self.chatgpt_model, + messages, + overrides, + self.get_response_token_limit(self.chatgpt_model, 1024), + should_stream, + ), + ) + extra_info.thoughts.append( + self.format_thought_step_for_chatcompletion( + title="Prompt to generate answer", + messages=messages, + overrides=overrides, + model=self.chatgpt_model, + deployment=self.chatgpt_deployment, + usage=None, + ) + ) + return (extra_info, chat_coroutine) + + async def run_search_approach( + self, messages: list[ChatCompletionMessageParam], overrides: dict[str, Any], auth_claims: dict[str, Any] + ): use_text_search = overrides.get("retrieval_mode") in ["text", "hybrid", None] use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False @@ -78,18 +137,12 @@ async def run_until_final_call( top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) - filter = self.build_filter(overrides, auth_claims) + search_index_filter = self.build_filter(overrides, auth_claims) original_user_query = messages[-1]["content"] if not isinstance(original_user_query, str): raise ValueError("The most recent message content must be a string.") - reasoning_model_support = self.GPT_REASONING_MODELS.get(self.chatgpt_model) - if reasoning_model_support and (not reasoning_model_support.streaming and should_stream): - raise Exception( - f"{self.chatgpt_model} does not support streaming. Please use a different model or disable streaming." - ) - query_messages = self.prompt_manager.render_prompt( self.query_rewrite_prompt, {"user_query": original_user_query, "past_messages": messages[:-1]} ) @@ -125,7 +178,7 @@ async def run_until_final_call( results = await self.search( top, query_text, - filter, + search_index_filter, vectors, use_text_search, use_vector_search, @@ -138,16 +191,6 @@ async def run_until_final_call( # STEP 3: Generate a contextual and content specific answer using the search results and chat history text_sources = self.get_sources_content(results, use_semantic_captions, use_image_citation=False) - messages = self.prompt_manager.render_prompt( - self.answer_prompt, - self.get_system_prompt_variables(overrides.get("prompt_template")) - | { - "include_follow_up_questions": bool(overrides.get("suggest_followup_questions")), - "past_messages": messages[:-1], - "user_query": original_user_query, - "text_sources": text_sources, - }, - ) extra_info = ExtraInfo( DataPoints(text=text_sources), @@ -169,7 +212,7 @@ async def run_until_final_call( "use_semantic_ranker": use_semantic_ranker, "use_query_rewriting": use_query_rewriting, "top": top, - "filter": filter, + "filter": search_index_filter, "use_vector_search": use_vector_search, "use_text_search": use_text_search, }, @@ -178,26 +221,58 @@ async def run_until_final_call( "Search results", [result.serialize_for_results() for result in results], ), - self.format_thought_step_for_chatcompletion( - title="Prompt to generate answer", - messages=messages, - overrides=overrides, - model=self.chatgpt_model, - deployment=self.chatgpt_deployment, - usage=None, - ), ], ) + return extra_info - chat_coroutine = cast( - Union[Awaitable[ChatCompletion], Awaitable[AsyncStream[ChatCompletionChunk]]], - self.create_chat_completion( - self.chatgpt_deployment, - self.chatgpt_model, - messages, - overrides, - self.get_response_token_limit(self.chatgpt_model, 1024), - should_stream, - ), + async def run_agentic_retrieval_approach( + self, + messages: list[ChatCompletionMessageParam], + overrides: dict[str, Any], + auth_claims: dict[str, Any], + ): + minimum_reranker_score = overrides.get("minimum_reranker_score", 0) + search_index_filter = self.build_filter(overrides, auth_claims) + top = overrides.get("top", 3) + max_subqueries = overrides.get("max_subqueries", 10) + # 50 is the amount of documents that the reranker can process per query + max_docs_for_reranker = max_subqueries * 50 + + response, results = await self.run_agentic_retrieval( + messages=messages, + agent_client=self.agent_client, + search_index_name=self.search_index_name, + top=top, + filter_add_on=search_index_filter, + minimum_reranker_score=minimum_reranker_score, + max_docs_for_reranker=max_docs_for_reranker, ) - return (extra_info, chat_coroutine) + + text_sources = self.get_sources_content(results, use_semantic_captions=False, use_image_citation=False) + + extra_info = ExtraInfo( + DataPoints(text=text_sources), + thoughts=[ + ThoughtStep( + "Use agentic retrieval", + messages, + { + "reranker_threshold": minimum_reranker_score, + "max_docs_for_reranker": max_docs_for_reranker, + "filter": search_index_filter, + }, + ), + ThoughtStep( + f"Agentic retrieval results (top {top})", + [result.serialize_for_results() for result in results], + { + "query_plan": ( + [activity.as_dict() for activity in response.activity] if response.activity else None + ), + "model": self.agent_model, + "deployment": self.agent_deployment, + }, + ), + ], + ) + return extra_info diff --git a/app/backend/approaches/retrievethenread.py b/app/backend/approaches/retrievethenread.py index f842002e1e..032fa1a90c 100644 --- a/app/backend/approaches/retrievethenread.py +++ b/app/backend/approaches/retrievethenread.py @@ -1,5 +1,6 @@ from typing import Any, Optional, cast +from azure.search.documents.agent.aio import KnowledgeAgentRetrievalClient from azure.search.documents.aio import SearchClient from azure.search.documents.models import VectorQuery from openai import AsyncOpenAI @@ -21,6 +22,10 @@ def __init__( self, *, search_client: SearchClient, + search_index_name: str, + agent_model: Optional[str], + agent_deployment: Optional[str], + agent_client: KnowledgeAgentRetrievalClient, auth_helper: AuthenticationHelper, openai_client: AsyncOpenAI, chatgpt_model: str, @@ -37,6 +42,10 @@ def __init__( reasoning_effort: Optional[str] = None, ): self.search_client = search_client + self.search_index_name = search_index_name + self.agent_model = agent_model + self.agent_deployment = agent_deployment + self.agent_client = agent_client self.chatgpt_deployment = chatgpt_deployment self.openai_client = openai_client self.auth_helper = auth_helper @@ -61,11 +70,57 @@ async def run( session_state: Any = None, context: dict[str, Any] = {}, ) -> dict[str, Any]: + overrides = context.get("overrides", {}) + auth_claims = context.get("auth_claims", {}) + use_agentic_retrieval = True if overrides.get("use_agentic_retrieval") else False q = messages[-1]["content"] if not isinstance(q, str): raise ValueError("The most recent message content must be a string.") - overrides = context.get("overrides", {}) - auth_claims = context.get("auth_claims", {}) + + if use_agentic_retrieval: + extra_info = await self.run_agentic_retrieval_approach(messages, overrides, auth_claims) + else: + extra_info = await self.run_search_approach(messages, overrides, auth_claims) + + # Process results + messages = self.prompt_manager.render_prompt( + self.answer_prompt, + self.get_system_prompt_variables(overrides.get("prompt_template")) + | {"user_query": q, "text_sources": extra_info.data_points.text}, + ) + + chat_completion = cast( + ChatCompletion, + await self.create_chat_completion( + self.chatgpt_deployment, + self.chatgpt_model, + messages=messages, + overrides=overrides, + response_token_limit=self.get_response_token_limit(self.chatgpt_model, 1024), + ), + ) + extra_info.thoughts.append( + self.format_thought_step_for_chatcompletion( + title="Prompt to generate answer", + messages=messages, + overrides=overrides, + model=self.chatgpt_model, + deployment=self.chatgpt_deployment, + usage=chat_completion.usage, + ) + ) + return { + "message": { + "content": chat_completion.choices[0].message.content, + "role": chat_completion.choices[0].message.role, + }, + "context": extra_info, + "session_state": session_state, + } + + async def run_search_approach( + self, messages: list[ChatCompletionMessageParam], overrides: dict[str, Any], auth_claims: dict[str, Any] + ): use_text_search = overrides.get("retrieval_mode") in ["text", "hybrid", None] use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False @@ -75,6 +130,7 @@ async def run( minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) filter = self.build_filter(overrides, auth_claims) + q = str(messages[-1]["content"]) # If retrieval mode includes vectors, compute an embedding for the query vectors: list[VectorQuery] = [] @@ -95,26 +151,9 @@ async def run( use_query_rewriting, ) - # Process results text_sources = self.get_sources_content(results, use_semantic_captions, use_image_citation=False) - messages = self.prompt_manager.render_prompt( - self.answer_prompt, - self.get_system_prompt_variables(overrides.get("prompt_template")) - | {"user_query": q, "text_sources": text_sources}, - ) - chat_completion = cast( - ChatCompletion, - await self.create_chat_completion( - self.chatgpt_deployment, - self.chatgpt_model, - messages=messages, - overrides=overrides, - response_token_limit=self.get_response_token_limit(self.chatgpt_model, 1024), - ), - ) - - extra_info = ExtraInfo( + return ExtraInfo( DataPoints(text=text_sources), thoughts=[ ThoughtStep( @@ -134,22 +173,57 @@ async def run( "Search results", [result.serialize_for_results() for result in results], ), - self.format_thought_step_for_chatcompletion( - title="Prompt to generate answer", - messages=messages, - overrides=overrides, - model=self.chatgpt_model, - deployment=self.chatgpt_deployment, - usage=chat_completion.usage, - ), ], ) - return { - "message": { - "content": chat_completion.choices[0].message.content, - "role": chat_completion.choices[0].message.role, - }, - "context": extra_info, - "session_state": session_state, - } + async def run_agentic_retrieval_approach( + self, + messages: list[ChatCompletionMessageParam], + overrides: dict[str, Any], + auth_claims: dict[str, Any], + ): + minimum_reranker_score = overrides.get("minimum_reranker_score", 0) + search_index_filter = self.build_filter(overrides, auth_claims) + top = overrides.get("top", 3) + max_subqueries = overrides.get("max_subqueries", 10) + # 50 is the amount of documents that the reranker can process per query + max_docs_for_reranker = max_subqueries * 50 + + response, results = await self.run_agentic_retrieval( + messages, + self.agent_client, + search_index_name=self.search_index_name, + top=top, + filter_add_on=search_index_filter, + minimum_reranker_score=minimum_reranker_score, + max_docs_for_reranker=max_docs_for_reranker, + ) + + text_sources = self.get_sources_content(results, use_semantic_captions=False, use_image_citation=False) + + extra_info = ExtraInfo( + DataPoints(text=text_sources), + thoughts=[ + ThoughtStep( + "Use agentic retrieval", + messages, + { + "reranker_threshold": minimum_reranker_score, + "max_docs_for_reranker": max_docs_for_reranker, + "filter": search_index_filter, + }, + ), + ThoughtStep( + f"Agentic retrieval results (top {top})", + [result.serialize_for_results() for result in results], + { + "query_plan": ( + [activity.as_dict() for activity in response.activity] if response.activity else None + ), + "model": self.agent_model, + "deployment": self.agent_deployment, + }, + ), + ], + ) + return extra_info diff --git a/app/backend/config.py b/app/backend/config.py index 5f3354f2da..443c0171fa 100644 --- a/app/backend/config.py +++ b/app/backend/config.py @@ -17,6 +17,7 @@ CONFIG_VECTOR_SEARCH_ENABLED = "vector_search_enabled" CONFIG_SEARCH_CLIENT = "search_client" CONFIG_OPENAI_CLIENT = "openai_client" +CONFIG_AGENT_CLIENT = "agent_client" CONFIG_INGESTER = "ingester" CONFIG_LANGUAGE_PICKER_ENABLED = "language_picker_enabled" CONFIG_SPEECH_INPUT_ENABLED = "speech_input_enabled" @@ -29,6 +30,7 @@ CONFIG_STREAMING_ENABLED = "streaming_enabled" CONFIG_CHAT_HISTORY_BROWSER_ENABLED = "chat_history_browser_enabled" CONFIG_CHAT_HISTORY_COSMOS_ENABLED = "chat_history_cosmos_enabled" +CONFIG_AGENTIC_RETRIEVAL_ENABLED = "agentic_retrieval" CONFIG_COSMOS_HISTORY_CLIENT = "cosmos_history_client" CONFIG_COSMOS_HISTORY_CONTAINER = "cosmos_history_container" CONFIG_COSMOS_HISTORY_VERSION = "cosmos_history_version" diff --git a/app/backend/prepdocs.py b/app/backend/prepdocs.py index f05e1e426b..f03baac0dc 100644 --- a/app/backend/prepdocs.py +++ b/app/backend/prepdocs.py @@ -46,16 +46,33 @@ def clean_key_if_exists(key: Union[str, None]) -> Union[str, None]: async def setup_search_info( - search_service: str, index_name: str, azure_credential: AsyncTokenCredential, search_key: Union[str, None] = None + search_service: str, + index_name: str, + azure_credential: AsyncTokenCredential, + use_agentic_retrieval: Union[bool, None] = None, + azure_openai_endpoint: Union[str, None] = None, + agent_name: Union[str, None] = None, + agent_max_output_tokens: Union[int, None] = None, + azure_openai_searchagent_deployment: Union[str, None] = None, + azure_openai_searchagent_model: Union[str, None] = None, + search_key: Union[str, None] = None, ) -> SearchInfo: search_creds: Union[AsyncTokenCredential, AzureKeyCredential] = ( azure_credential if search_key is None else AzureKeyCredential(search_key) ) + if use_agentic_retrieval and azure_openai_searchagent_model is None: + raise ValueError("Azure OpenAI SearchAgent model must be specified when using agentic retrieval.") return SearchInfo( endpoint=f"https://{search_service}.search.windows.net/", credential=search_creds, index_name=index_name, + agent_name=agent_name, + agent_max_output_tokens=agent_max_output_tokens, + use_agentic_retrieval=use_agentic_retrieval, + azure_openai_endpoint=azure_openai_endpoint, + azure_openai_searchagent_model=azure_openai_searchagent_model, + azure_openai_searchagent_deployment=azure_openai_searchagent_deployment, ) @@ -314,6 +331,7 @@ async def main(strategy: Strategy, setup_index: bool = True): use_gptvision = os.getenv("USE_GPT4V", "").lower() == "true" use_acls = os.getenv("AZURE_ENFORCE_ACCESS_CONTROL") is not None dont_use_vectors = os.getenv("USE_VECTORS", "").lower() == "false" + use_agentic_retrieval = os.getenv("USE_AGENTIC_RETRIEVAL", "").lower() == "true" use_content_understanding = os.getenv("USE_MEDIA_DESCRIBER_AZURE_CU", "").lower() == "true" # Use the current user identity to connect to Azure services. See infra/main.bicep for role assignments. @@ -334,10 +352,22 @@ async def main(strategy: Strategy, setup_index: bool = True): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) + openai_host = os.environ["OPENAI_HOST"] + # Check for incompatibility + # if openai host is not azure + if openai_host != "azure" and use_agentic_retrieval: + raise Exception("Agentic retrieval requires an Azure OpenAI chat completion service") + search_info = loop.run_until_complete( setup_search_info( search_service=os.environ["AZURE_SEARCH_SERVICE"], index_name=os.environ["AZURE_SEARCH_INDEX"], + use_agentic_retrieval=use_agentic_retrieval, + agent_name=os.getenv("AZURE_SEARCH_AGENT"), + agent_max_output_tokens=int(os.getenv("AZURE_SEARCH_AGENT_MAX_OUTPUT_TOKENS", 10000)), + azure_openai_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + azure_openai_searchagent_deployment=os.getenv("AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT"), + azure_openai_searchagent_model=os.getenv("AZURE_OPENAI_SEARCHAGENT_MODEL"), azure_credential=azd_credential, search_key=clean_key_if_exists(args.searchkey), ) diff --git a/app/backend/prepdocslib/searchmanager.py b/app/backend/prepdocslib/searchmanager.py index b544bbf52a..e6ca925e24 100644 --- a/app/backend/prepdocslib/searchmanager.py +++ b/app/backend/prepdocslib/searchmanager.py @@ -9,6 +9,10 @@ BinaryQuantizationCompression, HnswAlgorithmConfiguration, HnswParameters, + KnowledgeAgent, + KnowledgeAgentAzureOpenAIModel, + KnowledgeAgentRequestLimits, + KnowledgeAgentTargetIndex, RescoringOptions, SearchableField, SearchField, @@ -255,14 +259,16 @@ async def create_index(self): name=self.search_info.index_name, fields=fields, semantic_search=SemanticSearch( + default_configuration_name="default", configurations=[ SemanticConfiguration( name="default", prioritized_fields=SemanticPrioritizedFields( - title_field=None, content_fields=[SemanticField(field_name="content")] + title_field=SemanticField(field_name="sourcepage"), + content_fields=[SemanticField(field_name="content")], ), ) - ] + ], ), vector_search=VectorSearch( profiles=vector_search_profiles, @@ -331,6 +337,79 @@ async def create_index(self): existing_index.vector_search.algorithms.append(image_vector_algorithm) await search_index_client.create_or_update_index(existing_index) + if existing_index.semantic_search: + if not existing_index.semantic_search.default_configuration_name: + logger.info("Adding default semantic configuration to index %s", self.search_info.index_name) + existing_index.semantic_search.default_configuration_name = "default" + + if existing_index.semantic_search.configurations: + existing_semantic_config = existing_index.semantic_search.configurations[0] + if ( + existing_semantic_config.prioritized_fields + and existing_semantic_config.prioritized_fields.title_field + and not existing_semantic_config.prioritized_fields.title_field.field_name == "sourcepage" + ): + logger.info("Updating semantic configuration for index %s", self.search_info.index_name) + existing_semantic_config.prioritized_fields.title_field = SemanticField( + field_name="sourcepage" + ) + + if existing_index.vector_search is not None and ( + existing_index.vector_search.vectorizers is None + or len(existing_index.vector_search.vectorizers) == 0 + ): + if self.embeddings is not None and isinstance(self.embeddings, AzureOpenAIEmbeddingService): + logger.info("Adding vectorizer to search index %s", self.search_info.index_name) + existing_index.vector_search.vectorizers = [ + AzureOpenAIVectorizer( + vectorizer_name=f"{self.search_info.index_name}-vectorizer", + parameters=AzureOpenAIVectorizerParameters( + resource_url=self.embeddings.open_ai_endpoint, + deployment_name=self.embeddings.open_ai_deployment, + model_name=self.embeddings.open_ai_model_name, + ), + ) + ] + await search_index_client.create_or_update_index(existing_index) + + else: + logger.info( + "Can't add vectorizer to search index %s since no Azure OpenAI embeddings service is defined", + self.search_info, + ) + if self.search_info.use_agentic_retrieval and self.search_info.agent_name: + await self.create_agent() + + async def create_agent(self): + if self.search_info.agent_name: + logger.info(f"Creating search agent named {self.search_info.agent_name}") + + async with self.search_info.create_search_index_client() as search_index_client: + await search_index_client.create_or_update_agent( + agent=KnowledgeAgent( + name=self.search_info.agent_name, + target_indexes=[ + KnowledgeAgentTargetIndex( + index_name=self.search_info.index_name, default_include_reference_source_data=True + ) + ], + models=[ + KnowledgeAgentAzureOpenAIModel( + azure_open_ai_parameters=AzureOpenAIVectorizerParameters( + resource_url=self.search_info.azure_openai_endpoint, + deployment_name=self.search_info.azure_openai_searchagent_deployment, + model_name=self.search_info.azure_openai_searchagent_model, + ) + ) + ], + request_limits=KnowledgeAgentRequestLimits( + max_output_size=self.search_info.agent_max_output_tokens + ), + ) + ) + + logger.info("Agent %s created successfully", self.search_info.agent_name) + async def update_content( self, sections: list[Section], image_embeddings: Optional[list[list[float]]] = None, url: Optional[str] = None ): diff --git a/app/backend/prepdocslib/strategy.py b/app/backend/prepdocslib/strategy.py index e194dd64bd..05bc72804d 100644 --- a/app/backend/prepdocslib/strategy.py +++ b/app/backend/prepdocslib/strategy.py @@ -1,6 +1,6 @@ from abc import ABC from enum import Enum -from typing import Union +from typing import Optional, Union from azure.core.credentials import AzureKeyCredential from azure.core.credentials_async import AsyncTokenCredential @@ -16,10 +16,27 @@ class SearchInfo: To learn more, please visit https://learn.microsoft.com/azure/search/search-what-is-azure-search """ - def __init__(self, endpoint: str, credential: Union[AsyncTokenCredential, AzureKeyCredential], index_name: str): + def __init__( + self, + endpoint: str, + credential: Union[AsyncTokenCredential, AzureKeyCredential], + index_name: str, + use_agentic_retrieval: Optional[bool] = False, + agent_name: Optional[str] = None, + agent_max_output_tokens: Optional[int] = None, + azure_openai_searchagent_model: Optional[str] = None, + azure_openai_searchagent_deployment: Optional[str] = None, + azure_openai_endpoint: Optional[str] = None, + ): self.endpoint = endpoint self.credential = credential self.index_name = index_name + self.agent_name = agent_name + self.agent_max_output_tokens = agent_max_output_tokens + self.use_agentic_retrieval = use_agentic_retrieval + self.azure_openai_searchagent_model = azure_openai_searchagent_model + self.azure_openai_searchagent_deployment = azure_openai_searchagent_deployment + self.azure_openai_endpoint = azure_openai_endpoint def create_search_client(self) -> SearchClient: return SearchClient(endpoint=self.endpoint, index_name=self.index_name, credential=self.credential) diff --git a/app/backend/requirements.in b/app/backend/requirements.in index 52f5e23d38..ac889f2b9d 100644 --- a/app/backend/requirements.in +++ b/app/backend/requirements.in @@ -7,7 +7,7 @@ tenacity azure-ai-documentintelligence==1.0.0b4 azure-cognitiveservices-speech azure-cosmos -azure-search-documents==11.6.0b9 +azure-search-documents==11.6.0b12 azure-storage-blob azure-storage-file-datalake uvicorn diff --git a/app/backend/requirements.txt b/app/backend/requirements.txt index 6948289a96..0cd4981685 100644 --- a/app/backend/requirements.txt +++ b/app/backend/requirements.txt @@ -53,7 +53,7 @@ azure-monitor-opentelemetry==1.6.1 # via -r requirements.in azure-monitor-opentelemetry-exporter==1.0.0b32 # via azure-monitor-opentelemetry -azure-search-documents==11.6.0b11 +azure-search-documents==11.6.0b12 # via -r requirements.in azure-storage-blob==12.22.0 # via diff --git a/app/frontend/src/api/models.ts b/app/frontend/src/api/models.ts index 8cd37f7b7e..d6014ba25f 100644 --- a/app/frontend/src/api/models.ts +++ b/app/frontend/src/api/models.ts @@ -26,6 +26,7 @@ export type ChatAppRequestOverrides = { exclude_category?: string; seed?: number; top?: number; + max_subqueries?: number; temperature?: number; minimum_search_score?: number; minimum_reranker_score?: number; @@ -39,6 +40,7 @@ export type ChatAppRequestOverrides = { gpt4v_input?: GPT4VInput; vector_fields: VectorFields; language: string; + use_agentic_retrieval: boolean; }; export type ResponseMessage = { @@ -98,6 +100,7 @@ export type Config = { showSpeechOutputAzure: boolean; showChatHistoryBrowser: boolean; showChatHistoryCosmos: boolean; + showAgenticRetrievalOption: boolean; }; export type SimpleAPIResponse = { diff --git a/app/frontend/src/components/AnalysisPanel/AgentPlan.tsx b/app/frontend/src/components/AnalysisPanel/AgentPlan.tsx new file mode 100644 index 0000000000..21e246eb12 --- /dev/null +++ b/app/frontend/src/components/AnalysisPanel/AgentPlan.tsx @@ -0,0 +1,78 @@ +import React from "react"; +import { TokenUsageGraph, TokenUsage } from "./TokenUsageGraph"; +import { Light as SyntaxHighlighter } from "react-syntax-highlighter"; +import { a11yLight } from "react-syntax-highlighter/dist/esm/styles/hljs"; +import json from "react-syntax-highlighter/dist/esm/languages/hljs/json"; +import styles from "./AnalysisPanel.module.css"; +SyntaxHighlighter.registerLanguage("json", json); + +type ModelQueryPlanningStep = { + id: number; + type: "ModelQueryPlanning"; + input_tokens: number; + output_tokens: number; +}; + +type AzureSearchQueryStep = { + id: number; + type: "AzureSearchQuery"; + target_index: string; + query: { search: string }; + query_time: string; + count: number; + elapsed_ms: number; +}; + +type Step = ModelQueryPlanningStep | AzureSearchQueryStep; + +interface Props { + query_plan: Step[]; + description: any; +} + +export const AgentPlan: React.FC = ({ query_plan, description }) => { + // find the planning step + const planning = query_plan.find((step): step is ModelQueryPlanningStep => step.type === "ModelQueryPlanning"); + + // collect all search query steps + const queries = query_plan.filter((step): step is AzureSearchQueryStep => step.type === "AzureSearchQuery"); + + return ( +
+ {planning && ( + + )} + +
Subqueries
+ {queries.length > 0 && ( + + + + + + + + + + {queries.map(q => ( + + + + + + ))} + +
SubqueryTotal Result CountElapsed MS
{q.query.search}{q.count}{q.elapsed_ms}
+ )} +
+ ); +}; diff --git a/app/frontend/src/components/AnalysisPanel/AnalysisPanel.module.css b/app/frontend/src/components/AnalysisPanel/AnalysisPanel.module.css index 84b9f110ea..9dae51a8e8 100644 --- a/app/frontend/src/components/AnalysisPanel/AnalysisPanel.module.css +++ b/app/frontend/src/components/AnalysisPanel/AnalysisPanel.module.css @@ -44,6 +44,13 @@ margin-bottom: 0.5em; } +.tQuery { + color: #010207; + position: relative; + font-size: 0.875em; + margin-bottom: 0.5em; +} + .tCodeBlock { max-height: 18.75em; } @@ -134,3 +141,10 @@ background-color: #424242; color: #ffffff; } + +.subqueriesTable, +.subqueriesTable th, +.subqueriesTable td, +.subqueriesTable tr { + background: #fff; +} diff --git a/app/frontend/src/components/AnalysisPanel/ThoughtProcess.tsx b/app/frontend/src/components/AnalysisPanel/ThoughtProcess.tsx index f666960da1..196a87023f 100644 --- a/app/frontend/src/components/AnalysisPanel/ThoughtProcess.tsx +++ b/app/frontend/src/components/AnalysisPanel/ThoughtProcess.tsx @@ -7,6 +7,7 @@ import styles from "./AnalysisPanel.module.css"; import { Thoughts } from "../../api"; import { TokenUsageGraph } from "./TokenUsageGraph"; +import { AgentPlan } from "./AgentPlan"; SyntaxHighlighter.registerLanguage("json", json); @@ -23,13 +24,14 @@ export const ThoughtProcess = ({ thoughts }: Props) => {
{t.title}
{t.props && - (Object.keys(t.props).filter(k => k !== "token_usage") || []).map((k: any) => ( + (Object.keys(t.props).filter(k => k !== "token_usage" && k !== "query_plan") || []).map((k: any) => ( {k}: {JSON.stringify(t.props?.[k])} ))} {t.props?.token_usage && } + {t.props?.query_plan && } {Array.isArray(t.description) ? ( {JSON.stringify(t.description, null, 2)} diff --git a/app/frontend/src/components/Settings/Settings.tsx b/app/frontend/src/components/Settings/Settings.tsx index 1b5fc9e410..7d101abf36 100644 --- a/app/frontend/src/components/Settings/Settings.tsx +++ b/app/frontend/src/components/Settings/Settings.tsx @@ -14,6 +14,7 @@ export interface SettingsProps { promptTemplate: string; temperature: number; retrieveCount: number; + maxSubqueryCount: number; seed: number | null; minimumSearchScore: number; minimumRerankerScore: number; @@ -45,12 +46,15 @@ export interface SettingsProps { promptTemplatePrefix?: string; promptTemplateSuffix?: string; showSuggestFollowupQuestions?: boolean; + showAgenticRetrievalOption: boolean; + useAgenticRetrieval: boolean; } export const Settings = ({ promptTemplate, temperature, retrieveCount, + maxSubqueryCount, seed, minimumSearchScore, minimumRerankerScore, @@ -81,7 +85,9 @@ export const Settings = ({ useSuggestFollowupQuestions, promptTemplatePrefix, promptTemplateSuffix, - showSuggestFollowupQuestions + showSuggestFollowupQuestions, + showAgenticRetrievalOption, + useAgenticRetrieval }: SettingsProps) => { const { t } = useTranslation(); @@ -92,12 +98,16 @@ export const Settings = ({ const temperatureFieldId = useId("temperatureField"); const seedId = useId("seed"); const seedFieldId = useId("seedField"); + const agenticRetrievalId = useId("agenticRetrieval"); + const agenticRetrievalFieldId = useId("agenticRetrievalField"); const searchScoreId = useId("searchScore"); const searchScoreFieldId = useId("searchScoreField"); const rerankerScoreId = useId("rerankerScore"); const rerankerScoreFieldId = useId("rerankerScoreField"); const retrieveCountId = useId("retrieveCount"); const retrieveCountFieldId = useId("retrieveCountField"); + const maxSubqueryCountId = useId("maxSubqueryCount"); + const maxSubqueryCountFieldId = useId("maxSubqueryCountField"); const includeCategoryId = useId("includeCategory"); const includeCategoryFieldId = useId("includeCategoryField"); const excludeCategoryId = useId("excludeCategory"); @@ -160,18 +170,31 @@ export const Settings = ({ onRenderLabel={props => renderLabel(props, seedId, seedFieldId, t("helpTexts.seed"))} /> - onChange("minimumSearchScore", parseFloat(val || "0"))} - aria-labelledby={searchScoreId} - onRenderLabel={props => renderLabel(props, searchScoreId, searchScoreFieldId, t("helpTexts.searchScore"))} - /> + {showAgenticRetrievalOption && ( + onChange("useAgenticRetrieval", !!checked)} + aria-labelledby={agenticRetrievalId} + onRenderLabel={props => renderLabel(props, agenticRetrievalId, agenticRetrievalFieldId, t("helpTexts.suggestFollowupQuestions"))} + /> + )} + {!useAgenticRetrieval && !useGPT4V && ( + onChange("minimumSearchScore", parseFloat(val || "0"))} + aria-labelledby={searchScoreId} + onRenderLabel={props => renderLabel(props, searchScoreId, searchScoreFieldId, t("helpTexts.searchScore"))} + /> + )} {showSemanticRankerOption && ( )} + {showAgenticRetrievalOption && useAgenticRetrieval && ( + onChange("maxSubqueryCount", parseInt(val || "10"))} + aria-labelledby={maxSubqueryCountId} + onRenderLabel={props => renderLabel(props, maxSubqueryCountId, maxSubqueryCountFieldId, t("helpTexts.maxSubqueryCount"))} + /> + )} + renderLabel(props, excludeCategoryId, excludeCategoryFieldId, t("helpTexts.excludeCategory"))} /> - {showSemanticRankerOption && ( + {showSemanticRankerOption && !useAgenticRetrieval && ( <> )} - {showQueryRewritingOption && ( + {showQueryRewritingOption && !useAgenticRetrieval && ( <> )} - {showGPT4VOptions && ( + {showGPT4VOptions && !useAgenticRetrieval && ( )} - {showVectorOption && ( + {showVectorOption && !useAgenticRetrieval && ( (0); const [retrievalMode, setRetrievalMode] = useState(RetrievalMode.Hybrid); const [retrieveCount, setRetrieveCount] = useState(3); + const [maxSubqueryCount, setMaxSubqueryCount] = useState(10); const [useSemanticRanker, setUseSemanticRanker] = useState(true); const [useSemanticCaptions, setUseSemanticCaptions] = useState(false); const [useQueryRewriting, setUseQueryRewriting] = useState(false); @@ -54,6 +55,8 @@ export function Component(): JSX.Element { const [showSpeechOutputAzure, setShowSpeechOutputAzure] = useState(false); const audio = useRef(new Audio()).current; const [isPlaying, setIsPlaying] = useState(false); + const [showAgenticRetrievalOption, setShowAgenticRetrievalOption] = useState(false); + const [useAgenticRetrieval, setUseAgenticRetrieval] = useState(false); const lastQuestionRef = useRef(""); @@ -97,6 +100,8 @@ export function Component(): JSX.Element { setShowSpeechInput(config.showSpeechInput); setShowSpeechOutputBrowser(config.showSpeechOutputBrowser); setShowSpeechOutputAzure(config.showSpeechOutputAzure); + setShowAgenticRetrievalOption(config.showAgenticRetrievalOption); + setUseAgenticRetrieval(config.showAgenticRetrievalOption); }); }; @@ -130,6 +135,7 @@ export function Component(): JSX.Element { include_category: includeCategory.length === 0 ? undefined : includeCategory, exclude_category: excludeCategory.length === 0 ? undefined : excludeCategory, top: retrieveCount, + max_subqueries: maxSubqueryCount, temperature: temperature, minimum_reranker_score: minimumRerankerScore, minimum_search_score: minimumSearchScore, @@ -144,6 +150,7 @@ export function Component(): JSX.Element { use_gpt4v: useGPT4V, gpt4v_input: gpt4vInput, language: i18n.language, + use_agentic_retrieval: useAgenticRetrieval, ...(seed !== null ? { seed: seed } : {}) } }, @@ -186,6 +193,9 @@ export function Component(): JSX.Element { case "retrieveCount": setRetrieveCount(value); break; + case "maxSubqueryCount": + setMaxSubqueryCount(value); + break; case "useSemanticRanker": setUseSemanticRanker(value); break; @@ -222,6 +232,8 @@ export function Component(): JSX.Element { case "retrievalMode": setRetrievalMode(value); break; + case "useAgenticRetrieval": + setUseAgenticRetrieval(value); } }; @@ -334,6 +346,7 @@ export function Component(): JSX.Element { promptTemplateSuffix={promptTemplateSuffix} temperature={temperature} retrieveCount={retrieveCount} + maxSubqueryCount={maxSubqueryCount} seed={seed} minimumSearchScore={minimumSearchScore} minimumRerankerScore={minimumRerankerScore} @@ -357,6 +370,8 @@ export function Component(): JSX.Element { useLogin={!!useLogin} loggedIn={loggedIn} requireAccessControl={requireAccessControl} + showAgenticRetrievalOption={showAgenticRetrievalOption} + useAgenticRetrieval={useAgenticRetrieval} onChange={handleSettingsChange} /> {useLogin && } diff --git a/app/frontend/src/pages/chat/Chat.tsx b/app/frontend/src/pages/chat/Chat.tsx index d379b2a693..8813d5c4a7 100644 --- a/app/frontend/src/pages/chat/Chat.tsx +++ b/app/frontend/src/pages/chat/Chat.tsx @@ -46,6 +46,7 @@ const Chat = () => { const [minimumRerankerScore, setMinimumRerankerScore] = useState(0); const [minimumSearchScore, setMinimumSearchScore] = useState(0); const [retrieveCount, setRetrieveCount] = useState(3); + const [maxSubqueryCount, setMaxSubqueryCount] = useState(10); const [retrievalMode, setRetrievalMode] = useState(RetrievalMode.Hybrid); const [useSemanticRanker, setUseSemanticRanker] = useState(true); const [useQueryRewriting, setUseQueryRewriting] = useState(false); @@ -89,6 +90,9 @@ const Chat = () => { const [showSpeechOutputAzure, setShowSpeechOutputAzure] = useState(false); const [showChatHistoryBrowser, setShowChatHistoryBrowser] = useState(false); const [showChatHistoryCosmos, setShowChatHistoryCosmos] = useState(false); + const [showAgenticRetrievalOption, setShowAgenticRetrievalOption] = useState(false); + const [useAgenticRetrieval, setUseAgenticRetrieval] = useState(false); + const audio = useRef(new Audio()).current; const [isPlaying, setIsPlaying] = useState(false); @@ -129,6 +133,8 @@ const Chat = () => { setShowSpeechOutputAzure(config.showSpeechOutputAzure); setShowChatHistoryBrowser(config.showChatHistoryBrowser); setShowChatHistoryCosmos(config.showChatHistoryCosmos); + setShowAgenticRetrievalOption(config.showAgenticRetrievalOption); + setUseAgenticRetrieval(config.showAgenticRetrievalOption); }); }; @@ -224,6 +230,7 @@ const Chat = () => { use_gpt4v: useGPT4V, gpt4v_input: gpt4vInput, language: i18n.language, + use_agentic_retrieval: useAgenticRetrieval, ...(seed !== null ? { seed: seed } : {}) } }, @@ -302,6 +309,9 @@ const Chat = () => { case "retrieveCount": setRetrieveCount(value); break; + case "maxSubqueryCount": + setMaxSubqueryCount(value); + break; case "useSemanticRanker": setUseSemanticRanker(value); break; @@ -344,6 +354,8 @@ const Chat = () => { case "retrievalMode": setRetrievalMode(value); break; + case "useAgenticRetrieval": + setUseAgenticRetrieval(value); } }; @@ -521,6 +533,7 @@ const Chat = () => { promptTemplate={promptTemplate} temperature={temperature} retrieveCount={retrieveCount} + maxSubqueryCount={maxSubqueryCount} seed={seed} minimumSearchScore={minimumSearchScore} minimumRerankerScore={minimumRerankerScore} @@ -548,6 +561,8 @@ const Chat = () => { streamingEnabled={streamingEnabled} useSuggestFollowupQuestions={useSuggestFollowupQuestions} showSuggestFollowupQuestions={true} + showAgenticRetrievalOption={showAgenticRetrievalOption} + useAgenticRetrieval={useAgenticRetrieval} onChange={handleSettingsChange} /> {useLogin && } diff --git a/data/earth_at_night_508.pdf b/data/earth_at_night_508.pdf new file mode 100644 index 0000000000..9f71ca47f6 Binary files /dev/null and b/data/earth_at_night_508.pdf differ diff --git a/docs/agentic_retrieval.md b/docs/agentic_retrieval.md new file mode 100644 index 0000000000..70eee2403a --- /dev/null +++ b/docs/agentic_retrieval.md @@ -0,0 +1,56 @@ +# RAG chat: Using agentic retrieval + +This repository includes an optional feature that uses agentic retrieval to find the most relevant content given a user's conversation history. + +## Using the feature + +### Supported Models + +See the agentic retrieval documentation. + +### Prerequisites + +* A deployment of any of the supported agentic retrieval models in the [supported regions](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#standard-deployment-model-availability). If you're not sure, try to create a gpt-4o-mini deployment from your Azure OpenAI deployments page. + +### Deployment + +1. **Enable agentic retrieval:** + + Set the environment variables for your Azure OpenAI GPT deployments to your reasoning model + + ```shell + azd env set USE_AGENTIC_RETRIEVAL true + ``` + +2. **(Optional) Set the agentic retrieval model** + + You can configure which model agentic retrieval uses. By default, gpt-4o-mini is used + + For gpt-4o: + + ```shell + azd env set AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT searchagent + azd env set AZURE_OPENAI_SEARCHAGENT_MODEL gpt-4o + azd env set AZURE_OPENAI_SEARCHAGENT_MODEL_VERSION 2024-11-20 + ``` + +3. **Update the infrastructure and application:** + + Execute `azd up` to provision the infrastructure changes (only the new model, if you ran `up` previously) and deploy the application code with the updated environment variables. + +4. **Try out the feature:** + + Open the web app and start a new chat. Agentic retrieval will be used to find all sources. + +5. **Experiment with max subqueries:** + + Select the developer options in the web app and change max subqueries to any value between 1 and 20. This controls the maximum amount of subqueries that can be created in the query plan. + + ![Max subqueries screenshot](./images/max-subqueries.png) + +6. **Review the query plan** + + Agentic retrieval use additional billed tokens behind the scenes for the planning process. + To see the token usage, select the lightbulb icon on a chat answer. This will open the "Thought process" tab, which shows the amount of tokens used by and the queries produced by the planning process + + ![Thought process token usage](./images/query-plan.png) diff --git a/docs/azure_container_apps.md b/docs/azure_container_apps.md index be7b2a56b5..62d3a1a606 100644 --- a/docs/azure_container_apps.md +++ b/docs/azure_container_apps.md @@ -49,8 +49,8 @@ The default workload profile is Consumption. If you want to use a dedicated work azd env AZURE_CONTAINER_APPS_WORKLOAD_PROFILE D4 ``` -For a full list of workload profiles, please check [here](https://learn.microsoft.com/azure/container-apps/workload-profiles-overview#profile-types). -Please note dedicated workload profiles have a different billing model than Consumption plan. Please check [here](https://learn.microsoft.com/azure/container-apps/billing) for details. +For a full list of workload profiles, please check [the workload profile documentation](https://learn.microsoft.com/azure/container-apps/workload-profiles-overview#profile-types). +Please note dedicated workload profiles have a different billing model than Consumption plan. Please check [the billing documentation](https://learn.microsoft.com/azure/container-apps/billing) for details. ## Private endpoints diff --git a/docs/deploy_features.md b/docs/deploy_features.md index c402868932..c60a42a66f 100644 --- a/docs/deploy_features.md +++ b/docs/deploy_features.md @@ -128,6 +128,12 @@ This process does *not* delete your previous model deployment. If you want to de This feature allows you to use reasoning models to generate responses based on retrieved content. These models spend more time processing and understanding the user's request. To enable reasoning models, follow the steps in [the reasoning models guide](./reasoning.md). +## Using agentic retrieval + +⚠️ This feature is not currently compatible with [vision integration](./gpt4v.md). + +This feature allows you to use agentic retrieval in place of the Search API. To enable agentic retrieval, follow the steps in [the agentic retrieval guide](./agentic_retrieval.md) + ## Using different embedding models By default, the deployed Azure web app uses the `text-embedding-3-large` embedding model. If you want to use a different embedding model, you can do so by following these steps: diff --git a/docs/images/max-subqueries.png b/docs/images/max-subqueries.png new file mode 100644 index 0000000000..6740e8f076 Binary files /dev/null and b/docs/images/max-subqueries.png differ diff --git a/docs/images/query-plan.png b/docs/images/query-plan.png new file mode 100644 index 0000000000..2819244c05 Binary files /dev/null and b/docs/images/query-plan.png differ diff --git a/evals/requirements.txt b/evals/requirements.txt index 893de9de81..ccd1642830 100644 --- a/evals/requirements.txt +++ b/evals/requirements.txt @@ -3,4 +3,4 @@ rich ragas==0.2.13 rapidfuzz==3.12.1 langchain==0.3.17 -git+https://github.com/Azure-Samples/ai-rag-chat-evaluator.git@2025-05-01 +git+https://github.com/mattgotteiner/ai-rag-chat-evaluator@patch-1 diff --git a/infra/main.bicep b/infra/main.bicep index b63f0a10a4..2d9340e14b 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -24,6 +24,7 @@ param searchServiceLocation string = '' // Set in main.parameters.json @allowed(['free', 'basic', 'standard', 'standard2', 'standard3', 'storage_optimized_l1', 'storage_optimized_l2']) param searchServiceSkuName string // Set in main.parameters.json param searchIndexName string // Set in main.parameters.json +param searchAgentName string = useAgenticRetrieval ? '${searchIndexName}-agent' : '' param searchQueryLanguage string // Set in main.parameters.json param searchQuerySpeller string // Set in main.parameters.json param searchServiceSemanticRankerLevel string // Set in main.parameters.json @@ -39,6 +40,7 @@ param storageContainerName string = 'content' param storageSkuName string // Set in main.parameters.json param defaultReasoningEffort string // Set in main.parameters.json +param useAgenticRetrieval bool // Set in main.parameters.json param userStorageAccountName string = '' param userStorageContainerName string = 'user-content' @@ -149,7 +151,7 @@ param chatGptDeploymentCapacity int = 0 var chatGpt = { modelName: !empty(chatGptModelName) ? chatGptModelName : 'gpt-4o-mini' - deploymentName: !empty(chatGptDeploymentName) ? chatGptDeploymentName : 'gpt-4o-mini' + deploymentName: !empty(chatGptDeploymentName) ? chatGptDeploymentName : 'chat' deploymentVersion: !empty(chatGptDeploymentVersion) ? chatGptDeploymentVersion : '2024-07-18' deploymentSkuName: !empty(chatGptDeploymentSkuName) ? chatGptDeploymentSkuName : 'GlobalStandard' // Not backward-compatible deploymentCapacity: chatGptDeploymentCapacity != 0 ? chatGptDeploymentCapacity : 30 @@ -177,7 +179,7 @@ param gpt4vDeploymentSkuName string = '' param gpt4vDeploymentCapacity int = 0 var gpt4v = { modelName: !empty(gpt4vModelName) ? gpt4vModelName : 'gpt-4o' - deploymentName: !empty(gpt4vDeploymentName) ? gpt4vDeploymentName : 'gpt-4o' + deploymentName: !empty(gpt4vDeploymentName) ? gpt4vDeploymentName : 'vision' deploymentVersion: !empty(gpt4vModelVersion) ? gpt4vModelVersion : '2024-08-06' deploymentSkuName: !empty(gpt4vDeploymentSkuName) ? gpt4vDeploymentSkuName : 'GlobalStandard' // Not-backward compatible deploymentCapacity: gpt4vDeploymentCapacity != 0 ? gpt4vDeploymentCapacity : 10 @@ -190,12 +192,25 @@ param evalDeploymentSkuName string = '' param evalDeploymentCapacity int = 0 var eval = { modelName: !empty(evalModelName) ? evalModelName : 'gpt-4o' - deploymentName: !empty(evalDeploymentName) ? evalDeploymentName : 'gpt-4o' + deploymentName: !empty(evalDeploymentName) ? evalDeploymentName : 'eval' deploymentVersion: !empty(evalModelVersion) ? evalModelVersion : '2024-08-06' deploymentSkuName: !empty(evalDeploymentSkuName) ? evalDeploymentSkuName : 'GlobalStandard' // Not backward-compatible deploymentCapacity: evalDeploymentCapacity != 0 ? evalDeploymentCapacity : 30 } +param searchAgentModelName string = '' +param searchAgentDeploymentName string = '' +param searchAgentModelVersion string = '' +param searchAgentDeploymentSkuName string = '' +param searchAgentDeploymentCapacity int = 0 +var searchAgent = { + modelName: !empty(searchAgentModelName) ? searchAgentModelName : 'gpt-4o' + deploymentName: !empty(searchAgentDeploymentName) ? searchAgentDeploymentName : 'searchagent' + deploymentVersion: !empty(searchAgentModelVersion) ? searchAgentModelVersion : '2024-08-06' + deploymentSkuName: !empty(searchAgentDeploymentSkuName) ? searchAgentDeploymentSkuName : 'GlobalStandard' + deploymentCapacity: searchAgentDeploymentCapacity != 0 ? searchAgentDeploymentCapacity : 30 +} + param tenantId string = tenant().tenantId param authTenantId string = '' @@ -387,6 +402,7 @@ var appEnvVariables = { AZURE_STORAGE_ACCOUNT: storage.outputs.name AZURE_STORAGE_CONTAINER: storageContainerName AZURE_SEARCH_INDEX: searchIndexName + AZURE_SEARCH_AGENT: searchAgentName AZURE_SEARCH_SERVICE: searchService.outputs.name AZURE_SEARCH_SEMANTIC_RANKER: actualSearchServiceSemanticRankerLevel AZURE_SEARCH_QUERY_REWRITING: searchServiceQueryRewriting @@ -404,6 +420,7 @@ var appEnvVariables = { USE_SPEECH_INPUT_BROWSER: useSpeechInputBrowser USE_SPEECH_OUTPUT_BROWSER: useSpeechOutputBrowser USE_SPEECH_OUTPUT_AZURE: useSpeechOutputAzure + USE_AGENTIC_RETRIEVAL: useAgenticRetrieval // Chat history settings USE_CHAT_HISTORY_BROWSER: useChatHistoryBrowser USE_CHAT_HISTORY_COSMOS: useChatHistoryCosmos @@ -423,6 +440,8 @@ var appEnvVariables = { AZURE_OPENAI_CHATGPT_DEPLOYMENT: chatGpt.deploymentName AZURE_OPENAI_EMB_DEPLOYMENT: embedding.deploymentName AZURE_OPENAI_GPT4V_DEPLOYMENT: useGPT4V ? gpt4v.deploymentName : '' + AZURE_OPENAI_SEARCHAGENT_MODEL: searchAgent.modelName + AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT: searchAgent.deploymentName AZURE_OPENAI_API_VERSION: azureOpenAiApiVersion AZURE_OPENAI_API_KEY_OVERRIDE: azureOpenAiApiKey AZURE_OPENAI_CUSTOM_URL: azureOpenAiCustomUrl @@ -631,6 +650,22 @@ var openAiDeployments = concat( } } ] + : [], + useAgenticRetrieval + ? [ + { + name: searchAgent.deploymentName + model: { + format: 'OpenAI' + name: searchAgent.modelName + version: searchAgent.deploymentVersion + } + sku: { + name: searchAgent.deploymentSkuName + capacity: searchAgent.deploymentCapacity + } + } + ] : [] ) @@ -1034,7 +1069,7 @@ module openAiRoleBackend 'core/security/role.bicep' = if (isAzureOpenAiHost && d } } -module openAiRoleSearchService 'core/security/role.bicep' = if (isAzureOpenAiHost && deployAzureOpenAi && useIntegratedVectorization) { +module openAiRoleSearchService 'core/security/role.bicep' = if (isAzureOpenAiHost && deployAzureOpenAi && (useIntegratedVectorization || useAgenticRetrieval)) { scope: openAiResourceGroup name: 'openai-role-searchservice' params: { @@ -1260,6 +1295,7 @@ output AZURE_OPENAI_GPT4V_MODEL string = gpt4v.modelName // Specific to Azure OpenAI output AZURE_OPENAI_SERVICE string = isAzureOpenAiHost && deployAzureOpenAi ? openAi.outputs.name : '' +output AZURE_OPENAI_ENDPOINT string = isAzureOpenAiHost && deployAzureOpenAi ? openAi.outputs.endpoint : '' output AZURE_OPENAI_API_VERSION string = isAzureOpenAiHost ? azureOpenAiApiVersion : '' output AZURE_OPENAI_RESOURCE_GROUP string = isAzureOpenAiHost ? openAiResourceGroup.name : '' output AZURE_OPENAI_CHATGPT_DEPLOYMENT string = isAzureOpenAiHost ? chatGpt.deploymentName : '' @@ -1275,6 +1311,8 @@ output AZURE_OPENAI_EVAL_DEPLOYMENT string = isAzureOpenAiHost && useEval ? eval output AZURE_OPENAI_EVAL_DEPLOYMENT_VERSION string = isAzureOpenAiHost && useEval ? eval.deploymentVersion : '' output AZURE_OPENAI_EVAL_DEPLOYMENT_SKU string = isAzureOpenAiHost && useEval ? eval.deploymentSkuName : '' output AZURE_OPENAI_EVAL_MODEL string = isAzureOpenAiHost && useEval ? eval.modelName : '' +output AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT string = isAzureOpenAiHost && useAgenticRetrieval ? searchAgent.deploymentName : '' +output AZURE_OPENAI_SEARCHAGENT_MODEL string = isAzureOpenAiHost && useAgenticRetrieval ? searchAgent.modelName : '' output AZURE_OPENAI_REASONING_EFFORT string = defaultReasoningEffort output AZURE_SPEECH_SERVICE_ID string = useSpeechOutputAzure ? speech.outputs.resourceId : '' output AZURE_SPEECH_SERVICE_LOCATION string = useSpeechOutputAzure ? speech.outputs.location : '' @@ -1286,6 +1324,7 @@ output AZURE_DOCUMENTINTELLIGENCE_SERVICE string = documentIntelligence.outputs. output AZURE_DOCUMENTINTELLIGENCE_RESOURCE_GROUP string = documentIntelligenceResourceGroup.name output AZURE_SEARCH_INDEX string = searchIndexName +output AZURE_SEARCH_AGENT string = searchAgentName output AZURE_SEARCH_SERVICE string = searchService.outputs.name output AZURE_SEARCH_SERVICE_RESOURCE_GROUP string = searchServiceResourceGroup.name output AZURE_SEARCH_SEMANTIC_RANKER string = actualSearchServiceSemanticRankerLevel diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 84303f198f..1c34063020 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -170,6 +170,21 @@ "evalDeploymentCapacity":{ "value": "${AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY}" }, + "searchAgentModelName":{ + "value": "${AZURE_OPENAI_SEARCHAGENT_MODEL}" + }, + "searchAgentModelVersion":{ + "value": "${AZURE_OPENAI_SEARCHAGENT_MODEL_VERSION}" + }, + "searchAgentDeploymentName": { + "value": "${AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT}" + }, + "searchAgentDeploymentSkuName":{ + "value": "${AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT_SKU}" + }, + "searchAgentDeploymentCapacity":{ + "value": "${AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT_CAPACITY}" + }, "openAiHost": { "value": "${OPENAI_HOST=azure}" }, @@ -334,6 +349,9 @@ }, "useAiProject": { "value": "${USE_AI_PROJECT=false}" + }, + "useAgenticRetrieval": { + "value": "${USE_AGENTIC_RETRIEVAL=false}" } } } diff --git a/requirements-dev.txt b/requirements-dev.txt index f5bcf15bc8..c4a17953ac 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,4 +12,4 @@ pytest-snapshot pre-commit locust pip-tools -mypy==1.14.1 +mypy==1.14.1 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 4516c564ff..ebe32b3486 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,9 +10,14 @@ import msal import pytest import pytest_asyncio +from azure.search.documents.agent.aio import KnowledgeAgentRetrievalClient from azure.search.documents.aio import SearchClient from azure.search.documents.indexes.aio import SearchIndexClient -from azure.search.documents.indexes.models import SearchField, SearchIndex +from azure.search.documents.indexes.models import ( + KnowledgeAgent, + SearchField, + SearchIndex, +) from azure.storage.blob.aio import ContainerClient from openai.types import CompletionUsage, CreateEmbeddingResponse, Embedding from openai.types.chat import ChatCompletion, ChatCompletionChunk @@ -34,6 +39,7 @@ MockBlobClient, MockResponse, mock_computervision_response, + mock_retrieval_response, mock_speak_text_cancelled, mock_speak_text_failed, mock_speak_text_success, @@ -46,6 +52,7 @@ SearchField(name="groups", type="Collection(Edm.String)"), ], ) +MockAgent = KnowledgeAgent(name="test", models=[], target_indexes=[], request_limits=[]) async def mock_search(self, *args, **kwargs): @@ -53,6 +60,15 @@ async def mock_search(self, *args, **kwargs): return MockAsyncSearchResultsIterator(kwargs.get("search_text"), kwargs.get("vector_queries")) +async def mock_retrieve(self, *args, **kwargs): + retrieval_request = kwargs.get("retrieval_request") + assert retrieval_request is not None + assert retrieval_request.target_index_params is not None + assert len(retrieval_request.target_index_params) == 1 + self.filter = retrieval_request.target_index_params[0].filter_add_on + return mock_retrieval_response() + + @pytest.fixture def mock_azurehttp_calls(monkeypatch): def mock_post(*args, **kwargs): @@ -251,6 +267,16 @@ async def mock_get_index(*args, **kwargs): monkeypatch.setattr(SearchIndexClient, "get_index", mock_get_index) +@pytest.fixture +def mock_acs_agent(monkeypatch): + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieve) + + async def mock_get_agent(*args, **kwargs): + return MockAgent + + monkeypatch.setattr(SearchIndexClient, "get_agent", mock_get_agent) + + @pytest.fixture def mock_acs_search_filter(monkeypatch): monkeypatch.setattr(SearchClient, "search", mock_search) @@ -343,6 +369,37 @@ def mock_blob_container_client(monkeypatch): }, ] +agent_envs = [ + { + "OPENAI_HOST": "azure", + "AZURE_OPENAI_SERVICE": "test-openai-service", + "AZURE_OPENAI_CHATGPT_MODEL": "gpt-4o-mini", + "AZURE_OPENAI_CHATGPT_DEPLOYMENT": "gpt-4o-mini", + "AZURE_OPENAI_EMB_DEPLOYMENT": "test-ada", + "AZURE_OPENAI_SEARCHAGENT_MODEL": "gpt-4o-mini", + "AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT": "gpt-4o-mini", + "USE_AGENTIC_RETRIEVAL": "true", + } +] + +agent_auth_envs = [ + { + "OPENAI_HOST": "azure", + "AZURE_OPENAI_SERVICE": "test-openai-service", + "AZURE_OPENAI_CHATGPT_MODEL": "gpt-4o-mini", + "AZURE_OPENAI_CHATGPT_DEPLOYMENT": "gpt-4o-mini", + "AZURE_OPENAI_EMB_DEPLOYMENT": "test-ada", + "AZURE_OPENAI_SEARCHAGENT_MODEL": "gpt-4o-mini", + "AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT": "gpt-4o-mini", + "USE_AGENTIC_RETRIEVAL": "true", + "AZURE_USE_AUTHENTICATION": "true", + "AZURE_SERVER_APP_ID": "SERVER_APP", + "AZURE_SERVER_APP_SECRET": "SECRET", + "AZURE_CLIENT_APP_ID": "CLIENT_APP", + "AZURE_TENANT_ID": "TENANT_ID", + } +] + @pytest.fixture(params=envs, ids=["client0", "client1"]) def mock_env(monkeypatch, request): @@ -394,6 +451,54 @@ def mock_reasoning_env(monkeypatch, request): yield +@pytest.fixture(params=agent_envs, ids=["agent_client0"]) +def mock_agent_env(monkeypatch, request): + with mock.patch.dict(os.environ, clear=True): + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT", "test-storage-account") + monkeypatch.setenv("AZURE_STORAGE_CONTAINER", "test-storage-container") + monkeypatch.setenv("AZURE_STORAGE_RESOURCE_GROUP", "test-storage-rg") + monkeypatch.setenv("AZURE_SUBSCRIPTION_ID", "test-storage-subid") + monkeypatch.setenv("ENABLE_LANGUAGE_PICKER", "true") + monkeypatch.setenv("USE_SPEECH_INPUT_BROWSER", "true") + monkeypatch.setenv("USE_SPEECH_OUTPUT_AZURE", "true") + monkeypatch.setenv("AZURE_SEARCH_INDEX", "test-search-index") + monkeypatch.setenv("AZURE_SEARCH_AGENT", "test-search-agent") + monkeypatch.setenv("AZURE_SEARCH_SERVICE", "test-search-service") + monkeypatch.setenv("AZURE_SPEECH_SERVICE_ID", "test-id") + monkeypatch.setenv("AZURE_SPEECH_SERVICE_LOCATION", "eastus") + monkeypatch.setenv("ALLOWED_ORIGIN", "https://frontend.com") + for key, value in request.param.items(): + monkeypatch.setenv(key, value) + + with mock.patch("app.AzureDeveloperCliCredential") as mock_default_azure_credential: + mock_default_azure_credential.return_value = MockAzureCredential() + yield + + +@pytest.fixture(params=agent_auth_envs, ids=["agent_auth_client0"]) +def mock_agent_auth_env(monkeypatch, request): + with mock.patch.dict(os.environ, clear=True): + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT", "test-storage-account") + monkeypatch.setenv("AZURE_STORAGE_CONTAINER", "test-storage-container") + monkeypatch.setenv("AZURE_STORAGE_RESOURCE_GROUP", "test-storage-rg") + monkeypatch.setenv("AZURE_SUBSCRIPTION_ID", "test-storage-subid") + monkeypatch.setenv("ENABLE_LANGUAGE_PICKER", "true") + monkeypatch.setenv("USE_SPEECH_INPUT_BROWSER", "true") + monkeypatch.setenv("USE_SPEECH_OUTPUT_AZURE", "true") + monkeypatch.setenv("AZURE_SEARCH_INDEX", "test-search-index") + monkeypatch.setenv("AZURE_SEARCH_AGENT", "test-search-agent") + monkeypatch.setenv("AZURE_SEARCH_SERVICE", "test-search-service") + monkeypatch.setenv("AZURE_SPEECH_SERVICE_ID", "test-id") + monkeypatch.setenv("AZURE_SPEECH_SERVICE_LOCATION", "eastus") + monkeypatch.setenv("ALLOWED_ORIGIN", "https://frontend.com") + for key, value in request.param.items(): + monkeypatch.setenv(key, value) + + with mock.patch("app.AzureDeveloperCliCredential") as mock_default_azure_credential: + mock_default_azure_credential.return_value = MockAzureCredential() + yield + + @pytest_asyncio.fixture(scope="function") async def client( monkeypatch, @@ -432,6 +537,52 @@ async def reasoning_client( yield test_app.test_client() +@pytest_asyncio.fixture(scope="function") +async def agent_client( + monkeypatch, + mock_agent_env, + mock_openai_chatcompletion, + mock_openai_embedding, + mock_acs_search, + mock_acs_agent, + mock_blob_container_client, + mock_azurehttp_calls, +): + quart_app = app.create_app() + + async with quart_app.test_app() as test_app: + test_app.app.config.update({"TESTING": True}) + mock_openai_chatcompletion(test_app.app.config[app.CONFIG_OPENAI_CLIENT]) + mock_openai_embedding(test_app.app.config[app.CONFIG_OPENAI_CLIENT]) + yield test_app.test_client() + + +@pytest_asyncio.fixture(scope="function") +async def agent_auth_client( + monkeypatch, + mock_agent_auth_env, + mock_openai_chatcompletion, + mock_openai_embedding, + mock_acs_search, + mock_acs_agent, + mock_blob_container_client, + mock_azurehttp_calls, + mock_confidential_client_success, + mock_validate_token_success, + mock_list_groups_success, +): + quart_app = app.create_app() + + async with quart_app.test_app() as test_app: + test_app.app.config.update({"TESTING": True}) + mock_openai_chatcompletion(test_app.app.config[app.CONFIG_OPENAI_CLIENT]) + mock_openai_embedding(test_app.app.config[app.CONFIG_OPENAI_CLIENT]) + client = test_app.test_client() + client.config = quart_app.config + + yield client + + @pytest_asyncio.fixture(scope="function") async def client_with_expiring_token( monkeypatch, diff --git a/tests/mocks.py b/tests/mocks.py index 788823941c..fae3022f94 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -6,6 +6,15 @@ import openai.types from azure.cognitiveservices.speech import ResultReason from azure.core.credentials_async import AsyncTokenCredential +from azure.search.documents.agent.models import ( + KnowledgeAgentAzureSearchDocReference, + KnowledgeAgentMessage, + KnowledgeAgentMessageTextContent, + KnowledgeAgentModelQueryPlanningActivityRecord, + KnowledgeAgentRetrievalResponse, + KnowledgeAgentSearchActivityRecord, + KnowledgeAgentSearchActivityRecordQuery, +) from azure.search.documents.models import ( VectorQuery, ) @@ -208,6 +217,39 @@ def mock_computervision_response(): ) +def mock_retrieval_response(): + return KnowledgeAgentRetrievalResponse( + response=[ + KnowledgeAgentMessage( + role="assistant", + content=[ + KnowledgeAgentMessageTextContent( + text=r'[{"ref_id":0,"title":"Benefit_Options-2.pdf","content":"There is a whistleblower policy."}]' + ) + ], + ) + ], + activity=[ + KnowledgeAgentModelQueryPlanningActivityRecord(id=0, input_tokens=10, output_tokens=20, elapsed_ms=200), + KnowledgeAgentSearchActivityRecord( + id=1, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="whistleblower query"), + count=10, + elapsed_ms=50, + ), + ], + references=[ + KnowledgeAgentAzureSearchDocReference( + id=0, + activity_source=1, + doc_key="Benefit_Options-2.pdf", + source_data={"content": "There is a whistleblower policy.", "sourcepage": "Benefit_Options-2.pdf"}, + ) + ], + ) + + class MockAudio: def __init__(self, audio_data): self.audio_data = audio_data diff --git a/tests/snapshots/test_app/test_ask_prompt_template/client0/result.json b/tests/snapshots/test_app/test_ask_prompt_template/client0/result.json index 3f41dc08d3..f83f1d6f9e 100644 --- a/tests/snapshots/test_app/test_ask_prompt_template/client0/result.json +++ b/tests/snapshots/test_app/test_ask_prompt_template/client0/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_prompt_template/client1/result.json b/tests/snapshots/test_app/test_ask_prompt_template/client1/result.json index aa7163fe2a..ac939263bd 100644 --- a/tests/snapshots/test_app/test_ask_prompt_template/client1/result.json +++ b/tests/snapshots/test_app/test_ask_prompt_template/client1/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_prompt_template_concat/client0/result.json b/tests/snapshots/test_app/test_ask_prompt_template_concat/client0/result.json index a895c0f723..f77470bcc3 100644 --- a/tests/snapshots/test_app/test_ask_prompt_template_concat/client0/result.json +++ b/tests/snapshots/test_app/test_ask_prompt_template_concat/client0/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_prompt_template_concat/client1/result.json b/tests/snapshots/test_app/test_ask_prompt_template_concat/client1/result.json index a4b1ea96bd..5f45fc15f6 100644 --- a/tests/snapshots/test_app/test_ask_prompt_template_concat/client1/result.json +++ b/tests/snapshots/test_app/test_ask_prompt_template_concat/client1/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_rtr_hybrid/client0/result.json b/tests/snapshots/test_app/test_ask_rtr_hybrid/client0/result.json index d31d6dbde9..90b1d3ee4a 100644 --- a/tests/snapshots/test_app/test_ask_rtr_hybrid/client0/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_hybrid/client0/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_rtr_hybrid/client1/result.json b/tests/snapshots/test_app/test_ask_rtr_hybrid/client1/result.json index 14462a67bd..6b6a8268de 100644 --- a/tests/snapshots/test_app/test_ask_rtr_hybrid/client1/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_hybrid/client1/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_rtr_text/client0/result.json b/tests/snapshots/test_app/test_ask_rtr_text/client0/result.json index 672910c02f..b935f47728 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text/client0/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text/client0/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_rtr_text/client1/result.json b/tests/snapshots/test_app/test_ask_rtr_text/client1/result.json index f3168c117a..171edf3150 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text/client1/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text/client1/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_rtr_text_agent/agent_client0/result.json b/tests/snapshots/test_app/test_ask_rtr_text_agent/agent_client0/result.json new file mode 100644 index 0000000000..827d9ace6d --- /dev/null +++ b/tests/snapshots/test_app/test_ask_rtr_text_agent/agent_client0/result.json @@ -0,0 +1,104 @@ +{ + "context": { + "data_points": { + "images": null, + "text": [ + "Benefit_Options-2.pdf: There is a whistleblower policy." + ] + }, + "followup_questions": null, + "thoughts": [ + { + "description": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "props": { + "filter": null, + "max_docs_for_reranker": 500, + "reranker_threshold": 0 + }, + "title": "Use agentic retrieval" + }, + { + "description": [ + { + "captions": [], + "category": null, + "content": "There is a whistleblower policy.", + "groups": null, + "id": "Benefit_Options-2.pdf", + "oids": null, + "reranker_score": null, + "score": null, + "search_agent_query": "whistleblower query", + "sourcefile": null, + "sourcepage": "Benefit_Options-2.pdf" + } + ], + "props": { + "deployment": "gpt-4o-mini", + "model": "gpt-4o-mini", + "query_plan": [ + { + "elapsed_ms": 200, + "id": 0, + "input_tokens": 10, + "output_tokens": 20, + "type": "ModelQueryPlanning" + }, + { + "count": 10, + "elapsed_ms": 50, + "id": 1, + "query": { + "search": "whistleblower query" + }, + "target_index": "index", + "type": "AzureSearchQuery" + } + ] + }, + "title": "Agentic retrieval results (top 3)" + }, + { + "description": [ + { + "content": "You are an intelligent assistant helping Contoso Inc employees with their healthcare plan questions and employee handbook questions.\nUse 'you' to refer to the individual asking the questions even if they ask with 'I'.\nAnswer the following question using only the data provided in the sources below.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response.\nIf you cannot answer using the sources below, say you don't know. Use below example to answer", + "role": "system" + }, + { + "content": "What is the deductible for the employee plan for a visit to Overlake in Bellevue?\n\nSources:\ninfo1.txt: deductibles depend on whether you are in-network or out-of-network. In-network deductibles are $500 for employee and $1000 for family. Out-of-network deductibles are $1000 for employee and $2000 for family.\ninfo2.pdf: Overlake is in-network for the employee plan.\ninfo3.pdf: Overlake is the name of the area that includes a park and ride near Bellevue.\ninfo4.pdf: In-network institutions include Overlake, Swedish and others in the region.", + "role": "user" + }, + { + "content": "In-network deductibles are $500 for employee and $1000 for family [info1.txt] and Overlake is in-network for the employee plan [info2.pdf][info4.pdf].", + "role": "assistant" + }, + { + "content": "What is the capital of France?\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy.", + "role": "user" + } + ], + "props": { + "deployment": "gpt-4o-mini", + "model": "gpt-4o-mini", + "token_usage": { + "completion_tokens": 896, + "prompt_tokens": 23, + "reasoning_tokens": 0, + "total_tokens": 919 + } + }, + "title": "Prompt to generate answer" + } + ] + }, + "message": { + "content": "The capital of France is Paris. [Benefit_Options-2.pdf].", + "role": "assistant" + }, + "session_state": null +} \ No newline at end of file diff --git a/tests/snapshots/test_app/test_ask_rtr_text_agent_filter/agent_auth_client0/result.json b/tests/snapshots/test_app/test_ask_rtr_text_agent_filter/agent_auth_client0/result.json new file mode 100644 index 0000000000..909fbe1fda --- /dev/null +++ b/tests/snapshots/test_app/test_ask_rtr_text_agent_filter/agent_auth_client0/result.json @@ -0,0 +1,104 @@ +{ + "context": { + "data_points": { + "images": null, + "text": [ + "Benefit_Options-2.pdf: There is a whistleblower policy." + ] + }, + "followup_questions": null, + "thoughts": [ + { + "description": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "props": { + "filter": "category ne 'excluded' and (oids/any(g:search.in(g, 'OID_X')) or groups/any(g:search.in(g, 'GROUP_Y, GROUP_Z')))", + "max_docs_for_reranker": 500, + "reranker_threshold": 0 + }, + "title": "Use agentic retrieval" + }, + { + "description": [ + { + "captions": [], + "category": null, + "content": "There is a whistleblower policy.", + "groups": null, + "id": "Benefit_Options-2.pdf", + "oids": null, + "reranker_score": null, + "score": null, + "search_agent_query": "whistleblower query", + "sourcefile": null, + "sourcepage": "Benefit_Options-2.pdf" + } + ], + "props": { + "deployment": "gpt-4o-mini", + "model": "gpt-4o-mini", + "query_plan": [ + { + "elapsed_ms": 200, + "id": 0, + "input_tokens": 10, + "output_tokens": 20, + "type": "ModelQueryPlanning" + }, + { + "count": 10, + "elapsed_ms": 50, + "id": 1, + "query": { + "search": "whistleblower query" + }, + "target_index": "index", + "type": "AzureSearchQuery" + } + ] + }, + "title": "Agentic retrieval results (top 3)" + }, + { + "description": [ + { + "content": "You are an intelligent assistant helping Contoso Inc employees with their healthcare plan questions and employee handbook questions.\nUse 'you' to refer to the individual asking the questions even if they ask with 'I'.\nAnswer the following question using only the data provided in the sources below.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response.\nIf you cannot answer using the sources below, say you don't know. Use below example to answer", + "role": "system" + }, + { + "content": "What is the deductible for the employee plan for a visit to Overlake in Bellevue?\n\nSources:\ninfo1.txt: deductibles depend on whether you are in-network or out-of-network. In-network deductibles are $500 for employee and $1000 for family. Out-of-network deductibles are $1000 for employee and $2000 for family.\ninfo2.pdf: Overlake is in-network for the employee plan.\ninfo3.pdf: Overlake is the name of the area that includes a park and ride near Bellevue.\ninfo4.pdf: In-network institutions include Overlake, Swedish and others in the region.", + "role": "user" + }, + { + "content": "In-network deductibles are $500 for employee and $1000 for family [info1.txt] and Overlake is in-network for the employee plan [info2.pdf][info4.pdf].", + "role": "assistant" + }, + { + "content": "What is the capital of France?\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy.", + "role": "user" + } + ], + "props": { + "deployment": "gpt-4o-mini", + "model": "gpt-4o-mini", + "token_usage": { + "completion_tokens": 896, + "prompt_tokens": 23, + "reasoning_tokens": 0, + "total_tokens": 919 + } + }, + "title": "Prompt to generate answer" + } + ] + }, + "message": { + "content": "The capital of France is Paris. [Benefit_Options-2.pdf].", + "role": "assistant" + }, + "session_state": null +} \ No newline at end of file diff --git a/tests/snapshots/test_app/test_ask_rtr_text_filter/auth_client0/result.json b/tests/snapshots/test_app/test_ask_rtr_text_filter/auth_client0/result.json index 3226e2c741..690d363b98 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text_filter/auth_client0/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text_filter/auth_client0/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_rtr_text_filter_public_documents/auth_public_documents_client0/result.json b/tests/snapshots/test_app/test_ask_rtr_text_filter_public_documents/auth_public_documents_client0/result.json index 9bf2a82bc9..391cb4714a 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text_filter_public_documents/auth_public_documents_client0/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text_filter_public_documents/auth_public_documents_client0/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_rtr_text_semanticcaptions/client0/result.json b/tests/snapshots/test_app/test_ask_rtr_text_semanticcaptions/client0/result.json index 3a9c7d6936..e5ea25f3d2 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text_semanticcaptions/client0/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text_semanticcaptions/client0/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_rtr_text_semanticcaptions/client1/result.json b/tests/snapshots/test_app/test_ask_rtr_text_semanticcaptions/client1/result.json index 2db882b50a..d481a2f01e 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text_semanticcaptions/client1/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text_semanticcaptions/client1/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_rtr_text_semanticranker/client0/result.json b/tests/snapshots/test_app/test_ask_rtr_text_semanticranker/client0/result.json index 3e76befe60..4723627288 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text_semanticranker/client0/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text_semanticranker/client0/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_rtr_text_semanticranker/client1/result.json b/tests/snapshots/test_app/test_ask_rtr_text_semanticranker/client1/result.json index 371e79bf5e..58f164e257 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text_semanticranker/client1/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text_semanticranker/client1/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_vision/client0/result.json b/tests/snapshots/test_app/test_ask_vision/client0/result.json index e9d4bfdca2..a0b3899161 100644 --- a/tests/snapshots/test_app/test_ask_vision/client0/result.json +++ b/tests/snapshots/test_app/test_ask_vision/client0/result.json @@ -38,6 +38,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_ask_vision/client1/result.json b/tests/snapshots/test_app/test_ask_vision/client1/result.json index d130c6c6e6..76788017a6 100644 --- a/tests/snapshots/test_app/test_ask_vision/client1/result.json +++ b/tests/snapshots/test_app/test_ask_vision/client1/result.json @@ -35,6 +35,7 @@ "oids": null, "reranker_score": 3.1704962253570557, "score": 0.04972677677869797, + "search_agent_query": null, "sourcefile": "Financial Market Analysis Report 2023.pdf", "sourcepage": "Financial Market Analysis Report 2023-6.png" } diff --git a/tests/snapshots/test_app/test_chat_followup/client0/result.json b/tests/snapshots/test_app/test_chat_followup/client0/result.json index a87b79ac06..2ef9486287 100644 --- a/tests/snapshots/test_app/test_chat_followup/client0/result.json +++ b/tests/snapshots/test_app/test_chat_followup/client0/result.json @@ -78,6 +78,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_followup/client1/result.json b/tests/snapshots/test_app/test_chat_followup/client1/result.json index d918ed2af2..b2320dfa8b 100644 --- a/tests/snapshots/test_app/test_chat_followup/client1/result.json +++ b/tests/snapshots/test_app/test_chat_followup/client1/result.json @@ -79,6 +79,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_hybrid/client0/result.json b/tests/snapshots/test_app/test_chat_hybrid/client0/result.json index 6868aafff7..450afbce97 100644 --- a/tests/snapshots/test_app/test_chat_hybrid/client0/result.json +++ b/tests/snapshots/test_app/test_chat_hybrid/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_hybrid/client1/result.json b/tests/snapshots/test_app/test_chat_hybrid/client1/result.json index c093b39071..d940ea2104 100644 --- a/tests/snapshots/test_app/test_chat_hybrid/client1/result.json +++ b/tests/snapshots/test_app/test_chat_hybrid/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_hybrid_semantic_captions/client0/result.json b/tests/snapshots/test_app/test_chat_hybrid_semantic_captions/client0/result.json index 6a53d78bf3..0cd9a21ef2 100644 --- a/tests/snapshots/test_app/test_chat_hybrid_semantic_captions/client0/result.json +++ b/tests/snapshots/test_app/test_chat_hybrid_semantic_captions/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_hybrid_semantic_captions/client1/result.json b/tests/snapshots/test_app/test_chat_hybrid_semantic_captions/client1/result.json index 2e53c6b9b4..c8ef1612d1 100644 --- a/tests/snapshots/test_app/test_chat_hybrid_semantic_captions/client1/result.json +++ b/tests/snapshots/test_app/test_chat_hybrid_semantic_captions/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_hybrid_semantic_ranker/client0/result.json b/tests/snapshots/test_app/test_chat_hybrid_semantic_ranker/client0/result.json index 2bb5784a58..3cd84d9e06 100644 --- a/tests/snapshots/test_app/test_chat_hybrid_semantic_ranker/client0/result.json +++ b/tests/snapshots/test_app/test_chat_hybrid_semantic_ranker/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_hybrid_semantic_ranker/client1/result.json b/tests/snapshots/test_app/test_chat_hybrid_semantic_ranker/client1/result.json index bc8ef7155d..3033cdab41 100644 --- a/tests/snapshots/test_app/test_chat_hybrid_semantic_ranker/client1/result.json +++ b/tests/snapshots/test_app/test_chat_hybrid_semantic_ranker/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_prompt_template/client0/result.json b/tests/snapshots/test_app/test_chat_prompt_template/client0/result.json index 03d211fa4b..8cba036157 100644 --- a/tests/snapshots/test_app/test_chat_prompt_template/client0/result.json +++ b/tests/snapshots/test_app/test_chat_prompt_template/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_prompt_template/client1/result.json b/tests/snapshots/test_app/test_chat_prompt_template/client1/result.json index 98279625db..56fce921c9 100644 --- a/tests/snapshots/test_app/test_chat_prompt_template/client1/result.json +++ b/tests/snapshots/test_app/test_chat_prompt_template/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_prompt_template_concat/client0/result.json b/tests/snapshots/test_app/test_chat_prompt_template_concat/client0/result.json index cf0c5f3580..3be296c7a4 100644 --- a/tests/snapshots/test_app/test_chat_prompt_template_concat/client0/result.json +++ b/tests/snapshots/test_app/test_chat_prompt_template_concat/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_prompt_template_concat/client1/result.json b/tests/snapshots/test_app/test_chat_prompt_template_concat/client1/result.json index 8aa6a80f2a..e5bbd176ba 100644 --- a/tests/snapshots/test_app/test_chat_prompt_template_concat/client1/result.json +++ b/tests/snapshots/test_app/test_chat_prompt_template_concat/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_seed/client0/result.json b/tests/snapshots/test_app/test_chat_seed/client0/result.json index 6868aafff7..450afbce97 100644 --- a/tests/snapshots/test_app/test_chat_seed/client0/result.json +++ b/tests/snapshots/test_app/test_chat_seed/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_seed/client1/result.json b/tests/snapshots/test_app/test_chat_seed/client1/result.json index c093b39071..d940ea2104 100644 --- a/tests/snapshots/test_app/test_chat_seed/client1/result.json +++ b/tests/snapshots/test_app/test_chat_seed/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_session_state_persists/client0/result.json b/tests/snapshots/test_app/test_chat_session_state_persists/client0/result.json index 9cddd24798..a58d3390d3 100644 --- a/tests/snapshots/test_app/test_chat_session_state_persists/client0/result.json +++ b/tests/snapshots/test_app/test_chat_session_state_persists/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_session_state_persists/client1/result.json b/tests/snapshots/test_app/test_chat_session_state_persists/client1/result.json index 36b1d61f56..8c67d2d404 100644 --- a/tests/snapshots/test_app/test_chat_session_state_persists/client1/result.json +++ b/tests/snapshots/test_app/test_chat_session_state_persists/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_stream_followup/client0/result.jsonlines b/tests/snapshots/test_app/test_chat_stream_followup/client0/result.jsonlines index eeca662510..8380ef5141 100644 --- a/tests/snapshots/test_app/test_chat_stream_followup/client0/result.jsonlines +++ b/tests/snapshots/test_app/test_chat_stream_followup/client0/result.jsonlines @@ -1,5 +1,5 @@ -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini"}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini"}}], "followup_questions": null}, "session_state": null} {"delta": {"content": null, "role": "assistant"}} {"delta": {"content": "The capital of France is Paris. [Benefit_Options-2.pdf]. ", "role": "assistant"}} -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} -{"delta": {"role": "assistant"}, "context": {"context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "followup_questions": ["What is the capital of Spain?"]}} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "followup_questions": ["What is the capital of Spain?"]}} diff --git a/tests/snapshots/test_app/test_chat_stream_followup/client1/result.jsonlines b/tests/snapshots/test_app/test_chat_stream_followup/client1/result.jsonlines index d8215d797d..def8c8147b 100644 --- a/tests/snapshots/test_app/test_chat_stream_followup/client1/result.jsonlines +++ b/tests/snapshots/test_app/test_chat_stream_followup/client1/result.jsonlines @@ -1,5 +1,5 @@ -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt"}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt"}}], "followup_questions": null}, "session_state": null} {"delta": {"content": null, "role": "assistant"}} {"delta": {"content": "The capital of France is Paris. [Benefit_Options-2.pdf]. ", "role": "assistant"}} -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} -{"delta": {"role": "assistant"}, "context": {"context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "followup_questions": ["What is the capital of Spain?"]}} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n\n\n\n\nGenerate 3 very brief follow-up questions that the user would likely ask next.\nEnclose the follow-up questions in double angle brackets. Example:\n<>\n<>\n<>\nDo not repeat questions that have already been asked.\nMake sure the last question ends with \">>\"."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "followup_questions": ["What is the capital of Spain?"]}} diff --git a/tests/snapshots/test_app/test_chat_stream_session_state_persists/client0/result.jsonlines b/tests/snapshots/test_app/test_chat_stream_session_state_persists/client0/result.jsonlines index 750691cb87..1feee1e314 100644 --- a/tests/snapshots/test_app/test_chat_stream_session_state_persists/client0/result.jsonlines +++ b/tests/snapshots/test_app/test_chat_stream_session_state_persists/client0/result.jsonlines @@ -1,4 +1,4 @@ -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini"}}], "followup_questions": null}, "session_state": {"conversation_id": 1234}} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini"}}], "followup_questions": null}, "session_state": {"conversation_id": 1234}} {"delta": {"content": null, "role": "assistant"}} {"delta": {"content": "The capital of France is Paris. [Benefit_Options-2.pdf].", "role": null}} -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": {"conversation_id": 1234}} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": {"conversation_id": 1234}} diff --git a/tests/snapshots/test_app/test_chat_stream_session_state_persists/client1/result.jsonlines b/tests/snapshots/test_app/test_chat_stream_session_state_persists/client1/result.jsonlines index dea22c846e..78a5ba87b1 100644 --- a/tests/snapshots/test_app/test_chat_stream_session_state_persists/client1/result.jsonlines +++ b/tests/snapshots/test_app/test_chat_stream_session_state_persists/client1/result.jsonlines @@ -1,4 +1,4 @@ -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt"}}], "followup_questions": null}, "session_state": {"conversation_id": 1234}} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt"}}], "followup_questions": null}, "session_state": {"conversation_id": 1234}} {"delta": {"content": null, "role": "assistant"}} {"delta": {"content": "The capital of France is Paris. [Benefit_Options-2.pdf].", "role": null}} -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": {"conversation_id": 1234}} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": {"conversation_id": 1234}} diff --git a/tests/snapshots/test_app/test_chat_stream_text/client0/result.jsonlines b/tests/snapshots/test_app/test_chat_stream_text/client0/result.jsonlines index decd3d54ab..1921ccb193 100644 --- a/tests/snapshots/test_app/test_chat_stream_text/client0/result.jsonlines +++ b/tests/snapshots/test_app/test_chat_stream_text/client0/result.jsonlines @@ -1,4 +1,4 @@ -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini"}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini"}}], "followup_questions": null}, "session_state": null} {"delta": {"content": null, "role": "assistant"}} {"delta": {"content": "The capital of France is Paris. [Benefit_Options-2.pdf].", "role": null}} -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} diff --git a/tests/snapshots/test_app/test_chat_stream_text/client1/result.jsonlines b/tests/snapshots/test_app/test_chat_stream_text/client1/result.jsonlines index d98838102e..2a98d4730b 100644 --- a/tests/snapshots/test_app/test_chat_stream_text/client1/result.jsonlines +++ b/tests/snapshots/test_app/test_chat_stream_text/client1/result.jsonlines @@ -1,4 +1,4 @@ -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt"}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt"}}], "followup_questions": null}, "session_state": null} {"delta": {"content": null, "role": "assistant"}} {"delta": {"content": "The capital of France is Paris. [Benefit_Options-2.pdf].", "role": null}} -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} diff --git a/tests/snapshots/test_app/test_chat_stream_text_filter/auth_client0/result.jsonlines b/tests/snapshots/test_app/test_chat_stream_text_filter/auth_client0/result.jsonlines index 954e063ff0..e632b25868 100644 --- a/tests/snapshots/test_app/test_chat_stream_text_filter/auth_client0/result.jsonlines +++ b/tests/snapshots/test_app/test_chat_stream_text_filter/auth_client0/result.jsonlines @@ -1,4 +1,4 @@ -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": "category ne 'excluded' and (oids/any(g:search.in(g, 'OID_X')) or groups/any(g:search.in(g, 'GROUP_Y, GROUP_Z')))", "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt"}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": "category ne 'excluded' and (oids/any(g:search.in(g, 'OID_X')) or groups/any(g:search.in(g, 'GROUP_Y, GROUP_Z')))", "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt"}}], "followup_questions": null}, "session_state": null} {"delta": {"content": null, "role": "assistant"}} {"delta": {"content": "The capital of France is Paris. [Benefit_Options-2.pdf].", "role": null}} -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": "category ne 'excluded' and (oids/any(g:search.in(g, 'OID_X')) or groups/any(g:search.in(g, 'GROUP_Y, GROUP_Z')))", "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": "category ne 'excluded' and (oids/any(g:search.in(g, 'OID_X')) or groups/any(g:search.in(g, 'GROUP_Y, GROUP_Z')))", "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} diff --git a/tests/snapshots/test_app/test_chat_stream_text_reasoning/reasoning_client0/result.jsonlines b/tests/snapshots/test_app/test_chat_stream_text_reasoning/reasoning_client0/result.jsonlines index 5a907cf72b..0f669c2d39 100644 --- a/tests/snapshots/test_app/test_chat_stream_text_reasoning/reasoning_client0/result.jsonlines +++ b/tests/snapshots/test_app/test_chat_stream_text_reasoning/reasoning_client0/result.jsonlines @@ -1,4 +1,4 @@ -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": null}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": null}}], "followup_questions": null}, "session_state": null} {"delta": {"content": null, "role": "assistant"}} {"delta": {"content": "The capital of France is Paris. [Benefit_Options-2.pdf].", "role": null}} -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": null, "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": null, "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} diff --git a/tests/snapshots/test_app/test_chat_stream_text_reasoning/reasoning_client1/result.jsonlines b/tests/snapshots/test_app/test_chat_stream_text_reasoning/reasoning_client1/result.jsonlines index 346e8de618..13d920c6a6 100644 --- a/tests/snapshots/test_app/test_chat_stream_text_reasoning/reasoning_client1/result.jsonlines +++ b/tests/snapshots/test_app/test_chat_stream_text_reasoning/reasoning_client1/result.jsonlines @@ -1,4 +1,4 @@ -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low"}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low"}}], "followup_questions": null}, "session_state": null} {"delta": {"content": null, "role": "assistant"}} {"delta": {"content": "The capital of France is Paris. [Benefit_Options-2.pdf].", "role": null}} -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Benefit_Options-2.pdf: There is a whistleblower policy."], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: What is the capital of France?"}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "capital of France", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": false, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": "There is a whistleblower policy.", "category": null, "sourcepage": "Benefit_Options-2.pdf", "sourcefile": "Benefit_Options.pdf", "oids": null, "groups": null, "captions": [{"additional_properties": {}, "text": "Caption: A whistleblower policy.", "highlights": []}], "score": 0.03279569745063782, "reranker_score": 3.4577205181121826, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy."}], "props": {"model": "o3-mini", "deployment": "o3-mini", "reasoning_effort": "low", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 384, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} diff --git a/tests/snapshots/test_app/test_chat_stream_vision/client0/result.jsonlines b/tests/snapshots/test_app/test_chat_stream_vision/client0/result.jsonlines index 4b1b337845..6fcf1b40da 100644 --- a/tests/snapshots/test_app/test_chat_stream_vision/client0/result.jsonlines +++ b/tests/snapshots/test_app/test_chat_stream_vision/client0/result.jsonlines @@ -1,4 +1,4 @@ -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Financial Market Analysis Report 2023.pdf#page=6: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions "], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: Are interest rates high?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "interest rates", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Financial_Market_Analysis_Report_2023_pdf-46696E616E6369616C204D61726B657420416E616C79736973205265706F727420323032332E706466-page-14", "content": "31\nFinancial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors\nImpact of Interest Rates, Inflation, and GDP Growth on Financial Markets\n5\n4\n3\n2\n1\n0\n-1 2018 2019\n-2\n-3\n-4\n-5\n2020\n2021 2022 2023\nMacroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance.\n-Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends\nRelative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100)\n2028\nBased on historical data, current trends, and economic indicators, this section presents predictions ", "category": null, "sourcepage": "Financial Market Analysis Report 2023-6.png", "sourcefile": "Financial Market Analysis Report 2023.pdf", "oids": null, "groups": null, "captions": [], "score": 0.04972677677869797, "reranker_score": 3.1704962253570557}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "Are interest rates high?\n\nSources:\n\nFinancial Market Analysis Report 2023.pdf#page=6: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions"}], "props": {"model": "gpt-4o-mini"}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Financial Market Analysis Report 2023.pdf#page=6: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions "], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: Are interest rates high?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "interest rates", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Financial_Market_Analysis_Report_2023_pdf-46696E616E6369616C204D61726B657420416E616C79736973205265706F727420323032332E706466-page-14", "content": "31\nFinancial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors\nImpact of Interest Rates, Inflation, and GDP Growth on Financial Markets\n5\n4\n3\n2\n1\n0\n-1 2018 2019\n-2\n-3\n-4\n-5\n2020\n2021 2022 2023\nMacroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance.\n-Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends\nRelative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100)\n2028\nBased on historical data, current trends, and economic indicators, this section presents predictions ", "category": null, "sourcepage": "Financial Market Analysis Report 2023-6.png", "sourcefile": "Financial Market Analysis Report 2023.pdf", "oids": null, "groups": null, "captions": [], "score": 0.04972677677869797, "reranker_score": 3.1704962253570557, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "Are interest rates high?\n\nSources:\n\nFinancial Market Analysis Report 2023.pdf#page=6: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions"}], "props": {"model": "gpt-4o-mini"}}], "followup_questions": null}, "session_state": null} {"delta": {"content": null, "role": "assistant"}} {"delta": {"content": "The capital of France is Paris. [Benefit_Options-2.pdf].", "role": null}} -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Financial Market Analysis Report 2023.pdf#page=6: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions "], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: Are interest rates high?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "interest rates", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Financial_Market_Analysis_Report_2023_pdf-46696E616E6369616C204D61726B657420416E616C79736973205265706F727420323032332E706466-page-14", "content": "31\nFinancial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors\nImpact of Interest Rates, Inflation, and GDP Growth on Financial Markets\n5\n4\n3\n2\n1\n0\n-1 2018 2019\n-2\n-3\n-4\n-5\n2020\n2021 2022 2023\nMacroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance.\n-Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends\nRelative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100)\n2028\nBased on historical data, current trends, and economic indicators, this section presents predictions ", "category": null, "sourcepage": "Financial Market Analysis Report 2023-6.png", "sourcefile": "Financial Market Analysis Report 2023.pdf", "oids": null, "groups": null, "captions": [], "score": 0.04972677677869797, "reranker_score": 3.1704962253570557}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "Are interest rates high?\n\nSources:\n\nFinancial Market Analysis Report 2023.pdf#page=6: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Financial Market Analysis Report 2023.pdf#page=6: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions "], "images": null}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: Are interest rates high?"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}, {"title": "Search using generated search query", "description": "interest rates", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "use_vector_search": true, "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Financial_Market_Analysis_Report_2023_pdf-46696E616E6369616C204D61726B657420416E616C79736973205265706F727420323032332E706466-page-14", "content": "31\nFinancial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors\nImpact of Interest Rates, Inflation, and GDP Growth on Financial Markets\n5\n4\n3\n2\n1\n0\n-1 2018 2019\n-2\n-3\n-4\n-5\n2020\n2021 2022 2023\nMacroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance.\n-Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends\nRelative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100)\n2028\nBased on historical data, current trends, and economic indicators, this section presents predictions ", "category": null, "sourcepage": "Financial Market Analysis Report 2023-6.png", "sourcefile": "Financial Market Analysis Report 2023.pdf", "oids": null, "groups": null, "captions": [], "score": 0.04972677677869797, "reranker_score": 3.1704962253570557, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]."}, {"role": "user", "content": "Are interest rates high?\n\nSources:\n\nFinancial Market Analysis Report 2023.pdf#page=6: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions"}], "props": {"model": "gpt-4o-mini", "token_usage": {"prompt_tokens": 23, "completion_tokens": 896, "reasoning_tokens": 0, "total_tokens": 919}}}], "followup_questions": null}, "session_state": null} diff --git a/tests/snapshots/test_app/test_chat_stream_vision/client1/result.jsonlines b/tests/snapshots/test_app/test_chat_stream_vision/client1/result.jsonlines index d31541b4d7..1ec8a4d2c9 100644 --- a/tests/snapshots/test_app/test_chat_stream_vision/client1/result.jsonlines +++ b/tests/snapshots/test_app/test_chat_stream_vision/client1/result.jsonlines @@ -1,3 +1,3 @@ -{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Financial Market Analysis Report 2023-6.png: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions "], "images": ["data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="]}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: Are interest rates high?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt"}}, {"title": "Search using generated search query", "description": "interest rates", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "vector_fields": "textAndImageEmbeddings", "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Financial_Market_Analysis_Report_2023_pdf-46696E616E6369616C204D61726B657420416E616C79736973205265706F727420323032332E706466-page-14", "content": "31\nFinancial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors\nImpact of Interest Rates, Inflation, and GDP Growth on Financial Markets\n5\n4\n3\n2\n1\n0\n-1 2018 2019\n-2\n-3\n-4\n-5\n2020\n2021 2022 2023\nMacroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance.\n-Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends\nRelative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100)\n2028\nBased on historical data, current trends, and economic indicators, this section presents predictions ", "category": null, "sourcepage": "Financial Market Analysis Report 2023-6.png", "sourcefile": "Financial Market Analysis Report 2023.pdf", "oids": null, "groups": null, "captions": [], "score": 0.04972677677869797, "reranker_score": 3.1704962253570557}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "You are an intelligent assistant helping analyze the Annual Financial Report of Contoso Ltd., The documents contain text, graphs, tables and images.\nEach image source has the file name in the top left corner of the image with coordinates (10,10) pixels and is in the format SourceFileName:\nEach text source starts in a new line and has the file name followed by colon and the actual information\nAlways include the source name from the image or text for each fact you use in the response in the format: [filename]\nAnswer the following question using only the data provided in the sources below.\nIf asking a clarifying question to the user would help, ask the question.\nBe brief in your answers.\nThe text and image source can be the same file name, don't use the image title when citing the image source, only use the file name as mentioned\nIf you cannot answer using the sources below, say you don't know. Return just the answer without any input texts."}, {"role": "user", "content": [{"type": "text", "text": "Are interest rates high?"}, {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="}}, {"type": "text", "text": "Sources:\n\nFinancial Market Analysis Report 2023-6.png: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions"}]}], "props": {"model": "gpt-4"}}], "followup_questions": null}, "session_state": null} +{"delta": {"role": "assistant"}, "context": {"data_points": {"text": ["Financial Market Analysis Report 2023-6.png: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions "], "images": ["data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="]}, "thoughts": [{"title": "Prompt to generate search query", "description": [{"role": "system", "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0."}, {"role": "user", "content": "How did crypto do last year?"}, {"role": "assistant", "content": "Summarize Cryptocurrency Market Dynamics from last year"}, {"role": "user", "content": "What are my health plans?"}, {"role": "assistant", "content": "Show available health plans"}, {"role": "user", "content": "Generate search query for: Are interest rates high?"}], "props": {"model": "gpt-4o-mini", "deployment": "test-chatgpt"}}, {"title": "Search using generated search query", "description": "interest rates", "props": {"use_semantic_captions": false, "use_semantic_ranker": false, "use_query_rewriting": false, "top": 3, "filter": null, "vector_fields": "textAndImageEmbeddings", "use_text_search": true}}, {"title": "Search results", "description": [{"id": "file-Financial_Market_Analysis_Report_2023_pdf-46696E616E6369616C204D61726B657420416E616C79736973205265706F727420323032332E706466-page-14", "content": "31\nFinancial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors\nImpact of Interest Rates, Inflation, and GDP Growth on Financial Markets\n5\n4\n3\n2\n1\n0\n-1 2018 2019\n-2\n-3\n-4\n-5\n2020\n2021 2022 2023\nMacroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance.\n-Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends\nRelative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100)\n2028\nBased on historical data, current trends, and economic indicators, this section presents predictions ", "category": null, "sourcepage": "Financial Market Analysis Report 2023-6.png", "sourcefile": "Financial Market Analysis Report 2023.pdf", "oids": null, "groups": null, "captions": [], "score": 0.04972677677869797, "reranker_score": 3.1704962253570557, "search_agent_query": null}], "props": null}, {"title": "Prompt to generate answer", "description": [{"role": "system", "content": "You are an intelligent assistant helping analyze the Annual Financial Report of Contoso Ltd., The documents contain text, graphs, tables and images.\nEach image source has the file name in the top left corner of the image with coordinates (10,10) pixels and is in the format SourceFileName:\nEach text source starts in a new line and has the file name followed by colon and the actual information\nAlways include the source name from the image or text for each fact you use in the response in the format: [filename]\nAnswer the following question using only the data provided in the sources below.\nIf asking a clarifying question to the user would help, ask the question.\nBe brief in your answers.\nThe text and image source can be the same file name, don't use the image title when citing the image source, only use the file name as mentioned\nIf you cannot answer using the sources below, say you don't know. Return just the answer without any input texts."}, {"role": "user", "content": [{"type": "text", "text": "Are interest rates high?"}, {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="}}, {"type": "text", "text": "Sources:\n\nFinancial Market Analysis Report 2023-6.png: 31 Financial markets are interconnected, with movements in one segment often influencing others. This section examines the correlations between stock indices, cryptocurrency prices, and commodity prices, revealing how changes in one market can have ripple effects across the financial ecosystem.Impact of Macroeconomic Factors Impact of Interest Rates, Inflation, and GDP Growth on Financial Markets 5 4 3 2 1 0 -1 2018 2019 -2 -3 -4 -5 2020 2021 2022 2023 Macroeconomic factors such as interest rates, inflation, and GDP growth play a pivotal role in shaping financial markets. This section analyzes how these factors have influenced stock, cryptocurrency, and commodity markets over recent years, providing insights into the complex relationship between the economy and financial market performance. -Interest Rates % -Inflation Data % GDP Growth % :unselected: :unselected:Future Predictions and Trends Relative Growth Trends for S&P 500, Bitcoin, and Oil Prices (2024 Indexed to 100) 2028 Based on historical data, current trends, and economic indicators, this section presents predictions"}]}], "props": {"model": "gpt-4"}}], "followup_questions": null}, "session_state": null} {"delta": {"content": null, "role": "assistant"}} {"delta": {"content": "From the provided sources, the impact of interest rates and GDP growth on financial markets can be observed through the line graph. [Financial Market Analysis Report 2023-7.png]", "role": null}} diff --git a/tests/snapshots/test_app/test_chat_text/client0/result.json b/tests/snapshots/test_app/test_chat_text/client0/result.json index 5d42b2f04d..81d4d5f92a 100644 --- a/tests/snapshots/test_app/test_chat_text/client0/result.json +++ b/tests/snapshots/test_app/test_chat_text/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_text/client1/result.json b/tests/snapshots/test_app/test_chat_text/client1/result.json index 8d81c0e2e0..d5a1c19294 100644 --- a/tests/snapshots/test_app/test_chat_text/client1/result.json +++ b/tests/snapshots/test_app/test_chat_text/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_text_agent/agent_client0/result.json b/tests/snapshots/test_app/test_chat_text_agent/agent_client0/result.json new file mode 100644 index 0000000000..9e618d61b1 --- /dev/null +++ b/tests/snapshots/test_app/test_chat_text_agent/agent_client0/result.json @@ -0,0 +1,96 @@ +{ + "context": { + "data_points": { + "images": null, + "text": [ + "Benefit_Options-2.pdf: There is a whistleblower policy." + ] + }, + "followup_questions": null, + "thoughts": [ + { + "description": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "props": { + "filter": null, + "max_docs_for_reranker": 500, + "reranker_threshold": 0 + }, + "title": "Use agentic retrieval" + }, + { + "description": [ + { + "captions": [], + "category": null, + "content": "There is a whistleblower policy.", + "groups": null, + "id": "Benefit_Options-2.pdf", + "oids": null, + "reranker_score": null, + "score": null, + "search_agent_query": "whistleblower query", + "sourcefile": null, + "sourcepage": "Benefit_Options-2.pdf" + } + ], + "props": { + "deployment": "gpt-4o-mini", + "model": "gpt-4o-mini", + "query_plan": [ + { + "elapsed_ms": 200, + "id": 0, + "input_tokens": 10, + "output_tokens": 20, + "type": "ModelQueryPlanning" + }, + { + "count": 10, + "elapsed_ms": 50, + "id": 1, + "query": { + "search": "whistleblower query" + }, + "target_index": "index", + "type": "AzureSearchQuery" + } + ] + }, + "title": "Agentic retrieval results (top 3)" + }, + { + "description": [ + { + "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].", + "role": "system" + }, + { + "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy.", + "role": "user" + } + ], + "props": { + "deployment": "gpt-4o-mini", + "model": "gpt-4o-mini", + "token_usage": { + "completion_tokens": 896, + "prompt_tokens": 23, + "reasoning_tokens": 0, + "total_tokens": 919 + } + }, + "title": "Prompt to generate answer" + } + ] + }, + "message": { + "content": "The capital of France is Paris. [Benefit_Options-2.pdf].", + "role": "assistant" + }, + "session_state": null +} \ No newline at end of file diff --git a/tests/snapshots/test_app/test_chat_text_filter/auth_client0/result.json b/tests/snapshots/test_app/test_chat_text_filter/auth_client0/result.json index c20697d2f5..446ddfcecc 100644 --- a/tests/snapshots/test_app/test_chat_text_filter/auth_client0/result.json +++ b/tests/snapshots/test_app/test_chat_text_filter/auth_client0/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_text_filter_agent/agent_auth_client0/result.json b/tests/snapshots/test_app/test_chat_text_filter_agent/agent_auth_client0/result.json new file mode 100644 index 0000000000..40e7000667 --- /dev/null +++ b/tests/snapshots/test_app/test_chat_text_filter_agent/agent_auth_client0/result.json @@ -0,0 +1,96 @@ +{ + "context": { + "data_points": { + "images": null, + "text": [ + "Benefit_Options-2.pdf: There is a whistleblower policy." + ] + }, + "followup_questions": null, + "thoughts": [ + { + "description": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "props": { + "filter": "category ne 'excluded' and (oids/any(g:search.in(g, 'OID_X')) or groups/any(g:search.in(g, 'GROUP_Y, GROUP_Z')))", + "max_docs_for_reranker": 500, + "reranker_threshold": 0 + }, + "title": "Use agentic retrieval" + }, + { + "description": [ + { + "captions": [], + "category": null, + "content": "There is a whistleblower policy.", + "groups": null, + "id": "Benefit_Options-2.pdf", + "oids": null, + "reranker_score": null, + "score": null, + "search_agent_query": "whistleblower query", + "sourcefile": null, + "sourcepage": "Benefit_Options-2.pdf" + } + ], + "props": { + "deployment": "gpt-4o-mini", + "model": "gpt-4o-mini", + "query_plan": [ + { + "elapsed_ms": 200, + "id": 0, + "input_tokens": 10, + "output_tokens": 20, + "type": "ModelQueryPlanning" + }, + { + "count": 10, + "elapsed_ms": 50, + "id": 1, + "query": { + "search": "whistleblower query" + }, + "target_index": "index", + "type": "AzureSearchQuery" + } + ] + }, + "title": "Agentic retrieval results (top 3)" + }, + { + "description": [ + { + "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].", + "role": "system" + }, + { + "content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy.", + "role": "user" + } + ], + "props": { + "deployment": "gpt-4o-mini", + "model": "gpt-4o-mini", + "token_usage": { + "completion_tokens": 896, + "prompt_tokens": 23, + "reasoning_tokens": 0, + "total_tokens": 919 + } + }, + "title": "Prompt to generate answer" + } + ] + }, + "message": { + "content": "The capital of France is Paris. [Benefit_Options-2.pdf].", + "role": "assistant" + }, + "session_state": null +} \ No newline at end of file diff --git a/tests/snapshots/test_app/test_chat_text_filter_public_documents/auth_public_documents_client0/result.json b/tests/snapshots/test_app/test_chat_text_filter_public_documents/auth_public_documents_client0/result.json index 201fb9fcc8..8bf512a76e 100644 --- a/tests/snapshots/test_app/test_chat_text_filter_public_documents/auth_public_documents_client0/result.json +++ b/tests/snapshots/test_app/test_chat_text_filter_public_documents/auth_public_documents_client0/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_text_reasoning/reasoning_client0/result.json b/tests/snapshots/test_app/test_chat_text_reasoning/reasoning_client0/result.json index 0634d404ad..016bd920e2 100644 --- a/tests/snapshots/test_app/test_chat_text_reasoning/reasoning_client0/result.json +++ b/tests/snapshots/test_app/test_chat_text_reasoning/reasoning_client0/result.json @@ -78,6 +78,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_text_reasoning/reasoning_client1/result.json b/tests/snapshots/test_app/test_chat_text_reasoning/reasoning_client1/result.json index 431258417f..17836861aa 100644 --- a/tests/snapshots/test_app/test_chat_text_reasoning/reasoning_client1/result.json +++ b/tests/snapshots/test_app/test_chat_text_reasoning/reasoning_client1/result.json @@ -78,6 +78,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_text_semantic_ranker/client0/result.json b/tests/snapshots/test_app/test_chat_text_semantic_ranker/client0/result.json index 6740ce36cb..bd98db5ab5 100644 --- a/tests/snapshots/test_app/test_chat_text_semantic_ranker/client0/result.json +++ b/tests/snapshots/test_app/test_chat_text_semantic_ranker/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_text_semantic_ranker/client1/result.json b/tests/snapshots/test_app/test_chat_text_semantic_ranker/client1/result.json index 78d46952af..833040bf44 100644 --- a/tests/snapshots/test_app/test_chat_text_semantic_ranker/client1/result.json +++ b/tests/snapshots/test_app/test_chat_text_semantic_ranker/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_text_semanticcaptions/client0/result.json b/tests/snapshots/test_app/test_chat_text_semanticcaptions/client0/result.json index 694304c70d..c429280aaf 100644 --- a/tests/snapshots/test_app/test_chat_text_semanticcaptions/client0/result.json +++ b/tests/snapshots/test_app/test_chat_text_semanticcaptions/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_text_semanticcaptions/client1/result.json b/tests/snapshots/test_app/test_chat_text_semanticcaptions/client1/result.json index 51ceabdab8..161c91c4b6 100644 --- a/tests/snapshots/test_app/test_chat_text_semanticcaptions/client1/result.json +++ b/tests/snapshots/test_app/test_chat_text_semanticcaptions/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_text_semanticranker/client0/result.json b/tests/snapshots/test_app/test_chat_text_semanticranker/client0/result.json index 6740ce36cb..bd98db5ab5 100644 --- a/tests/snapshots/test_app/test_chat_text_semanticranker/client0/result.json +++ b/tests/snapshots/test_app/test_chat_text_semanticranker/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_text_semanticranker/client1/result.json b/tests/snapshots/test_app/test_chat_text_semanticranker/client1/result.json index 78d46952af..833040bf44 100644 --- a/tests/snapshots/test_app/test_chat_text_semanticranker/client1/result.json +++ b/tests/snapshots/test_app/test_chat_text_semanticranker/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_vector/client0/result.json b/tests/snapshots/test_app/test_chat_vector/client0/result.json index e07471425e..d70b6adeaf 100644 --- a/tests/snapshots/test_app/test_chat_vector/client0/result.json +++ b/tests/snapshots/test_app/test_chat_vector/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_vector/client1/result.json b/tests/snapshots/test_app/test_chat_vector/client1/result.json index 049906c697..ddf12b26c6 100644 --- a/tests/snapshots/test_app/test_chat_vector/client1/result.json +++ b/tests/snapshots/test_app/test_chat_vector/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_vector_semantic_ranker/client0/result.json b/tests/snapshots/test_app/test_chat_vector_semantic_ranker/client0/result.json index 2ff04cc1df..eeec58d14e 100644 --- a/tests/snapshots/test_app/test_chat_vector_semantic_ranker/client0/result.json +++ b/tests/snapshots/test_app/test_chat_vector_semantic_ranker/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_vector_semantic_ranker/client1/result.json b/tests/snapshots/test_app/test_chat_vector_semantic_ranker/client1/result.json index 0d1d59e299..6401c93de4 100644 --- a/tests/snapshots/test_app/test_chat_vector_semantic_ranker/client1/result.json +++ b/tests/snapshots/test_app/test_chat_vector_semantic_ranker/client1/result.json @@ -77,6 +77,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_vision/client0/result.json b/tests/snapshots/test_app/test_chat_vision/client0/result.json index 70e52d24b0..05330f4c2a 100644 --- a/tests/snapshots/test_app/test_chat_vision/client0/result.json +++ b/tests/snapshots/test_app/test_chat_vision/client0/result.json @@ -70,6 +70,7 @@ "oids": null, "reranker_score": 3.1704962253570557, "score": 0.04972677677869797, + "search_agent_query": null, "sourcefile": "Financial Market Analysis Report 2023.pdf", "sourcepage": "Financial Market Analysis Report 2023-6.png" } diff --git a/tests/snapshots/test_app/test_chat_vision/client1/result.json b/tests/snapshots/test_app/test_chat_vision/client1/result.json index bbbd57898c..a080480d45 100644 --- a/tests/snapshots/test_app/test_chat_vision/client1/result.json +++ b/tests/snapshots/test_app/test_chat_vision/client1/result.json @@ -67,6 +67,7 @@ "oids": null, "reranker_score": 3.1704962253570557, "score": 0.04972677677869797, + "search_agent_query": null, "sourcefile": "Financial Market Analysis Report 2023.pdf", "sourcepage": "Financial Market Analysis Report 2023-6.png" } diff --git a/tests/snapshots/test_app/test_chat_vision_vectors/client0/result.json b/tests/snapshots/test_app/test_chat_vision_vectors/client0/result.json index 3ed411512b..932e1aa371 100644 --- a/tests/snapshots/test_app/test_chat_vision_vectors/client0/result.json +++ b/tests/snapshots/test_app/test_chat_vision_vectors/client0/result.json @@ -76,6 +76,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_vision_vectors/client1/result.json b/tests/snapshots/test_app/test_chat_vision_vectors/client1/result.json index 1c91558a9d..fc622bcca7 100644 --- a/tests/snapshots/test_app/test_chat_vision_vectors/client1/result.json +++ b/tests/snapshots/test_app/test_chat_vision_vectors/client1/result.json @@ -67,6 +67,7 @@ "oids": null, "reranker_score": 3.1704962253570557, "score": 0.04972677677869797, + "search_agent_query": null, "sourcefile": "Financial Market Analysis Report 2023.pdf", "sourcepage": "Financial Market Analysis Report 2023-6.png" } diff --git a/tests/snapshots/test_app/test_chat_with_history/client0/result.json b/tests/snapshots/test_app/test_chat_with_history/client0/result.json index aa0c192e34..866d9f287d 100644 --- a/tests/snapshots/test_app/test_chat_with_history/client0/result.json +++ b/tests/snapshots/test_app/test_chat_with_history/client0/result.json @@ -84,6 +84,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_app/test_chat_with_history/client1/result.json b/tests/snapshots/test_app/test_chat_with_history/client1/result.json index 0b2eda9f3e..fa161f267f 100644 --- a/tests/snapshots/test_app/test_chat_with_history/client1/result.json +++ b/tests/snapshots/test_app/test_chat_with_history/client1/result.json @@ -85,6 +85,7 @@ "oids": null, "reranker_score": 3.4577205181121826, "score": 0.03279569745063782, + "search_agent_query": null, "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } diff --git a/tests/snapshots/test_prepdocslib_textsplitter/test_sentencetextsplitter_list_parse_and_split/text_splitter_sections.txt b/tests/snapshots/test_prepdocslib_textsplitter/test_sentencetextsplitter_list_parse_and_split/text_splitter_sections.txt index c2ec2c5795..cd6d5441f4 100644 --- a/tests/snapshots/test_prepdocslib_textsplitter/test_sentencetextsplitter_list_parse_and_split/text_splitter_sections.txt +++ b/tests/snapshots/test_prepdocslib_textsplitter/test_sentencetextsplitter_list_parse_and_split/text_splitter_sections.txt @@ -550,6 +550,168 @@ " PerksPlus is not only \ndesigned to support employees' physical health, but also their mental health. Regular exercise has been \nshown to reduce stress, improve mood, and enhance overall well -being. With PerksPlus, employees can \ninvest in their health and wel lness, while enjoying the peace of mind that comes with knowing they are \ngetting the support they need to lead a healthy life. \nWhat is Covered? \nPerksPlus covers a wide range of fitness activities, including but not limited to: \n\u2022 Gym memberships \n\u2022 Personal training sessions \n\u2022 Yoga and Pilates classes \n\u2022 Fitness equipment purchases \n\u2022 Sports team fees \n\u2022 Health retreats and spas \n\u2022 Outdoor adventure activities (such as rock climbing, hiking, and kayaking) \n\u2022 Group fitness classes (such as dance, martial arts, and cycling) \n\u2022 Virtual fitness programs (such as online yoga and workout classes) \nIn addition to the wide range of fitness activities covered by PerksPlus, the program also covers a variety \nof lessons and experiences that promote health and wellness. Some of the lessons covered under \nPerksPlus include: \n\u2022 Skiing and snowboarding lessons \n\u2022 Scuba diving lessons \n\u2022 Surfing lessons \n\u2022 Horseback ", " Some of the lessons covered under \nPerksPlus include: \n\u2022 Skiing and snowboarding lessons \n\u2022 Scuba diving lessons \n\u2022 Surfing lessons \n\u2022 Horseback riding lessons \nThese lessons provide employees with the opportunity to try new things, challenge themselves, and \nimprove their physical skills. They are also a great way to relieve stress and have fun while staying active. \nWith Perk sPlus, employees can choose from a variety of fitness programs to suit their individual needs \nand preferences. Whether you're looking to improve your physical fitness, reduce stress, or just have \nsome fun, PerksPlus has you covered. \nWhat is Not Covered? \nIn addition to the wid e range of activities covered by PerksPlus, there is also a list of things that are not \ncovered under the program. These include but are not limited to: \n\u2022 Non -fitness related expenses \n\u2022 Medical treatments and procedures \n\u2022 Travel expenses (unless rela ted to a fitness program) \u2022 Food and supplements \n " ], + "earth_at_night_508.pdf": [ + "Earth at NightNational Aeronautics and\nSpace Administration\nOur Planet in Brilliant Darknessiv Earth at Night\nCover image: This global view of Earth\u2019s city lights is a composite assembled from data acquired by the Suomi National Polar-orbiting \nPartnership (NPP) satellite. The data were acquired over nine days in April 2012 and 13 days in October 2012. \n(Image credit: NASA\u2019s Earth Observatory/NOAA/DOD)\nNP-2019-07-2739-HQ v\nContents\nForeword . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . x\nPreface . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . xii\nAcknowledgments . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . xiv\nAn Introduction to Nightlights . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 2\nBlue Marble Becomes Black Marble . . . . . . . . . . . . . . . . . . . . . .", + " . . . . . . . . . 2\nBlue Marble Becomes Black Marble . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 4\nVision and Remote Sensing . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 6\nSeeing Is Sensing . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 6\nWhat Is Light? . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 6\nRemote Sensing . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 8\nOrbiting Tools to Observe Nightlights . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 10\nImprovements in ISS Photography\u2014Northeastern United States . . . . . . . . . . . . . . . . 16\nComing into Focus\u2014Cairo, Egypt . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 18\nWhere Do Nightlights Come From? .", + " . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 18\nWhere Do Nightlights Come From? . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 20\nBeyond City Lights\u2014Java Sea . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 22\nMoonlight, Old Lights, and New Lights\u2014Gulf of Mexico . . . . . . . . . . . . . . . . . . . . . . . 24\nA World of Change . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 26\nNature\u2019s Light Shows . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 28\nFires . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 32\nTracking Fires Day and Night\u2014Northwest United States . . . . . . . . . . . . . . . . . . . . . . . 32\nProgression of the Rim Fire\u2014Yosemite, California . . . . . . . . . . . . . . . . . . . . . . . . . . . . 35\nSpreading Like Wildfire\u2014Yakutia, Siberia, Russia .", + " . . . . . . . . . . . . . . . . . . . . . . 35\nSpreading Like Wildfire\u2014Yakutia, Siberia, Russia . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 36\nCity Lights, or Not\u2014Australia . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 39\nBillowing Smoke in the Night \u2014Idaho and Montana . . . . . . . . . . . . . . . . . . . . . . . . . . . 40\nFires Light Up Mount Vesuvius\u2014Naples, Italy . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 42\nVolcanoes . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 44\nThe Infrared Glows of Kilauea\u2019s Lava Flows\u2014Hawaii . . . . . . . . . . . . . . . . . . . . . . . . . . 44\nMount Tongariro Erupts\u2014New Zealand . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 46vi Earth at Night\nNighttime Glow at Mount Etna\u2014Italy . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .", + " 46vi Earth at Night\nNighttime Glow at Mount Etna\u2014Italy . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 48\nMount Etna Erupts\u2014Italy . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 50\nMoonlight and Moonglint . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 52\nMoon Phases\u2014Persian Gulf . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 52\nMoonglint\u2014Elba and the Mediterranean . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 54\nClouds and Lightning . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 56\nMarine Layer Clouds\u2014California . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 56\nA Blizzard by Moonlight\u2014Northeastern United States . . . . . . . . . . . . . . . . . . . . . . . . . 59\nSensing Lightning from the ISS\u2014Saudi Arabia and Bolivia . . . .", + " . . . . . . . . . . . . . . . 59\nSensing Lightning from the ISS\u2014Saudi Arabia and Bolivia . . . . . . . . . . . . . . . . . . . . . 60\nLightning\u2014Japan . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 62\nElusive Sprite Captured from the ISS\u2014Southeast Asia . . . . . . . . . . . . . . . . . . . . . . . . 64\nThe Electric Eye of Cyclone Bansi\u2014Indian Ocean . . . . . . . . . . . . . . . . . . . . . . . . . . . . 67\nSnow and Ice . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 68\nSnow Cover\u2014Great Lakes Region, United States . . . . . . . . . . . . . . . . . . . . . . . . . . . . 68\nPolar Darkness\u2014The Arctic . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 71\nAuroras . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 72\nAurora Borealis\u2014Midwestern United States .", + " . . . . . . . . . . . . . . . . . . . . . . . . . . 72\nAurora Borealis\u2014Midwestern United States . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 72\nLooking Down at the Aurora\u2014Northern Hemisphere . . . . . . . . . . . . . . . . . . . . . . . . . . 74\nAurora Lights Up the Night\u2014Antarctica . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 76\nAirglow . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 78\nAirglow\u2014Australia . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 78\nNight Colors\u2014Russia . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 81\nWaves in Airglow\u2014Texas . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 82\nHuman Light Sources . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .", + " 82\nHuman Light Sources . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 84\nUrban Structure . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 86\nConstrained by Geography\u2014Reno, Nevada . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 86\nGrid of City Blocks\u2014Phoenix, Arizona . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 88\nUnpopulated Slopes of an Active Volcano\u2014Naples, Italy . . . . . . . . . . . . . . . . . . . . . . . 90\nDazzling Coastlines\u2014Italy . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 92\nLiving on Fertile Land\u2014Nile River, Egypt . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 94 vii\nThe Winding Seine River and the City of Light\u2014Paris, France . . . . . . . . . . . . . . . . . . . 96\nLighting Paths to Oil\u2014Qatar . . . . . . . . . . . . . . . . . . . . . . . . . . . . .", + " . . . . . 96\nLighting Paths to Oil\u2014Qatar . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 98\nSnaking Along Canyon Cliffs\u2014Haifa, Israel . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 100\nDifferences in Socio-Economic Strategies\u2014Korean Peninsula . . . . . . . . . . . . . . . . . 102\nNow You See Them, Now You Don\u2019t\u2014Argentina . . . . . . . . . . . . . . . . . . . . . . . . . . . . 104\nA Well-Lit Border, Indus River\u2014Pakistan . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 106\nUrban Development . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 108\nLighting Paths\u2014Across the United States . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 108\nThe Brightest Spot on Earth\u2014Las Vegas, Nevada . . . . . . . . . . . . . . . . . . . . . . . . . . . 110\nRapid Urban Growth\u2014Shanghai, China . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .", + " 110\nRapid Urban Growth\u2014Shanghai, China . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 112\nTurning Up the Lights\u2014India . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 114\nOlympics at Night\u2014Sochi, Russia . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 116\nPower Outages . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 118\n Hurricane Maria . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 118\nHurricane Maria into the Night\u2014Puerto Rico . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 118\nLights Out\u2014Puerto Rico . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 120\n Hurricane Matthew . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 122\nHurricane Matthew Brushes Florida Coast\u2014Southeast United States .", + " . . . . . . . . . . . . . . 122\nHurricane Matthew Brushes Florida Coast\u2014Southeast United States . . . . . . . . . . . . 122\nLights Out After Matthew\u2014Southeast United States . . . . . . . . . . . . . . . . . . . . . . . . . 124\n Hurricane Michael . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 126\nFlorida Slammed by Hurricane Michael\u2014Florida Panhandle . . . . . . . . . . . . . . . . . . . 126\nMichael Churns at Night\u2014Florida Panhandle . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 127\nLights Out in Michael\u2019s Wake\u2014Florida Panhandle . . . . . . . . . . . . . . . . . . . . . . . . . . . 128\n Hurricane Sandy . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 130\nOvernight View of Hurricane Sandy\u2014Eastern United States . . . . . . . . . . . . . . . . . . . 130\nBlackout\u2014New Jersey and New York . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .", + " 130\nBlackout\u2014New Jersey and New York . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 132\n Other Power Outages . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 134\nLights Out in Hatteras\u2014North Carolina . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 134\nRare Derecho Causes Power Outages\u2014Washington, D .C . . . . . . . . . . . . . . . . . . . . . 136\nEarthquake Aftermath\u2014Nepal . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 138viii Earth at Night\nEffects of War . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 140\nConflict in the Middle East\u2014Syria . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .140\nNightlights Change in the Middle East\u2014Syria and Iraq . . . . . . . . . . . . . . . . . . . . . . .142\nMining . . . . . . . . . . . . . . . . . . . . . . . . . .", + " . . . . . . . . . . . . . . . . . .142\nMining . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 144\nShale Revolution: As Clear as Night and Day\u2014South Texas . . . . . . . . . . . . . . . . . . .144\nTen Percent of the World\u2019s Gas Flares in One Spot\u2014Nigeria . . . . . . . . . . . . . . . . . . .146\nGas Flares in Bah\u00eda de Campeche\u2014Gulf of Mexico . . . . . . . . . . . . . . . . . . . . . . . . . .148\nGas Drilling\u2014North Dakota . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .150\nConnection Between Gas Flaring and Arctic Pollution\u2014North Dakota . . . . . . . . . . .152\nSea-Going Vessels . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 154\nSomething Fishy in the Atlantic Night\u2014South Atlantic Ocean . . . . . . . . . . . . . . . . . .154\nKorea and the Yellow Sea\u2014Korean Peninsula . . . . . . . . . . . . . . . . . .", + " . . . . . . . .154\nKorea and the Yellow Sea\u2014Korean Peninsula . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .156\nHoliday Lights . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 158\nBursting with Holiday Energy\u2014United States . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .158\nThe Lights of Ramadan and Eid al-Fitr\u2014Middle East . . . . . . . . . . . . . . . . . . . . . . . . .164\nEpilogue . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .168\nAppendix A: NASA\u2019s Black Marble Product Suite . . . . . . . . . . . . . . . . . . . . . . . . . .170\nAppendix B: Making a Cloud-Free, Global, Earth-at-Night Image Using NASA\u2019s \n Black Marble Product Suite . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .173\nAppendix C: OLS and VIIRS Technical Details . . . . . . . . . . . . . . . . . . . . . . . . . . . .176\nAppendix D: Additional Credits and Information .", + " . . . . . . . . . . . . . . . . . . . . . . .176\nAppendix D: Additional Credits and Information . . . . . . . . . . . . . . . . . . . . . . . . . . .179 ix\nx Earth at NightForeword\nNASA\u2019s Earth at Night explores the brilliance of our planet when it is in darkness. \n It is a compilation of stories depicting the interactions between science and \nwonder, and I am pleased to share this visually stunning and captivating exploration of \nour home planet.\nFrom space, our Earth looks tranquil. The blue ethereal vastness of the oceans \nharmoniously shares the space with verdant green land\u2014an undercurrent of gentle-ness and solitude. But spending time gazing at the images presented in this book, our home planet at night instantly reveals a different reality. Beautiful, filled with glow-ing communities, natural wonders, and striking illumination, our world is bustling with activity and life.\nDarkness is not void of illumination. It is the contrast, the area between light and \ndark, that is often the most illustrative.", + "\nDarkness is not void of illumination. It is the contrast, the area between light and \ndark, that is often the most illustrative. Darkness reminds me of where I came from and where I am now\u2014from a small town in the mountains, to the unique vantage point of the Nation\u2019s capital. Darkness is where dreamers and learners of all ages peer into the universe and think of questions about themselves and their space in the cosmos. Light is where they work, where they gather, and take time together.\nNASA\u2019s spacefaring satellites have compiled an unprecedented record of our \nEarth, and its luminescence in darkness, to captivate and spark curiosity. These missions see the contrast between dark and light through the lenses of scientific instruments. Our home planet is full of complex and dynamic cycles and processes. These soaring observers show us new ways to discern the nuances of light created by natural and human-made sources, such as auroras, wildfires, cities, phytoplankton, and volcanoes. xi\nScience not only changes what we know, but also how we think about our place \nin the cosmos.", + "show us new ways to discern the nuances of light created by natural and human-made sources, such as auroras, wildfires, cities, phytoplankton, and volcanoes. xi\nScience not only changes what we know, but also how we think about our place \nin the cosmos. I invite you to take a moment to discover our world at night through \nthe eyes of space science. I have no doubt that inherent beauty of these images will \ninspire\u2014feeding the soul and mind of the reader. As Vincent van Gogh said, \u201cFor my \npart I know nothing with any certainty, but the sight of the stars makes me dream.\u201d \nI challenge you to look back from the stars to see Earth in a completely new way.\nThomas H . Zurbuchen\nAssociate Administrator\nNASA Science Mission Directoratexii Earth at NightPreface\nTo keen observers, the nocturnal Earth is not pitch black, featureless, or static. \nThe stars and the Moon provide illumination that differs from, and complements, \ndaylight. Natural Earth processes such as volcanic eruptions, auroras, lightning, and meteors entering the atmosphere generate localized visible light on timescales ranging from subsecond (lightning), to days, weeks (forest fires), and months (volcanic eruptions).", + " Natural Earth processes such as volcanic eruptions, auroras, lightning, and meteors entering the atmosphere generate localized visible light on timescales ranging from subsecond (lightning), to days, weeks (forest fires), and months (volcanic eruptions).\nMost interesting and unique (as far as we know) to Earth, is the nighttime visible \nillumination emitted from our planet that is associated with human activities. Whether purposefully designed to banish darkness (such as lighting for safety, industrial activities, commerce, and transportation) or a secondary result of (such as gas flares associated with mining and hydrocarbon extraction activities, or nocturnal commercial fishing), anthro-pogenic sources of nighttime light are often broadly distributed in space and sustained in time\u2014over years and even decades. Because these light sources are inextricably tied to human activities and societies, extensive and long-term measurement and monitoring of Earth\u2019s anthropogenic nocturnal lights can provide valuable insights into the spatial distribution of our species and the ways in which society is changing\u2014and is changed by\u2014the environment on a wide range of time scales.", + "nocturnal lights can provide valuable insights into the spatial distribution of our species and the ways in which society is changing\u2014and is changed by\u2014the environment on a wide range of time scales.\nOver the past four decades, sensitive imaging instruments have been operated on \nlow-Earth-orbiting satellites to measure natural and human-caused visible nocturnal illumination, both reflected and Earth-generated. The satellite sensors provide unique imagery: global coverage yet with high spatial resolution, and frequent measurements over long periods of time.\nThe combined, multisatellite global nocturnal illumination dataset contains a \ntreasure trove of unique information about our planet and our species\u2014and the xiii\ninteractions between society and natural processes. Beyond academic study, Earth \nnightlight measurements are being used to help save lives and property around the \nglobe, by allowing accurate identification and monitoring of ongoing events like erup -\ntions and fires even in remote locations, and by pinpointing and enabling quantitative \ntracking of regions of power outage and recovery following extreme weather events and \ngeohazards in populated areas.", + " by allowing accurate identification and monitoring of ongoing events like erup -\ntions and fires even in remote locations, and by pinpointing and enabling quantitative \ntracking of regions of power outage and recovery following extreme weather events and \ngeohazards in populated areas.\nEarth at Night tells\u2014in the words of the women and men, the scientists and \nengineers who are actually designing the instruments and conducting the analyses \nof Earth\u2019s nocturnal illumination imagery\u2014the story of satellite measurements of global \nlight in the night. It shows how ever-increasing instrument capability has improved the \nsensitivity, accuracy, coverage, and resolution of the observations. Through striking \nillustrations and clear explanations, the book summarizes many examples of analyses \nfrom the satellite nightlight data record\u2014examples that themselves shine light on the \never-changing environment and our human impact on Earth.\nThis artful volume reaffirms our human ability to harness technology and science to \nobserve and understand Earth for the benefit of all humankind. Above all, it once again \nilluminates the beauty and majesty of our home planet\u2014at all hours.\nMichael H .", + " Above all, it once again \nilluminates the beauty and majesty of our home planet\u2014at all hours.\nMichael H . Freilich\nFormer Director, Earth Science Divisi on\n(October 23, 2010 \u2013 February 28, 2019)\nNASA Science Mission Directoratexiv Earth at Night\nAcknowledgments\nThe images presented and discussed here are merely the surface results of \nyears of work by thousands of scientists, engineers, technologists, outreach \npersonnel, and representatives of many disciplines. While the material is in the public \ndomain and free to use (as it comes from NASA funding), much of the information \npresented in this volume was developed by the staff of NASA\u2019s Earth Observatory \nwebsite ( https:// earthobservatory .nasa .gov , search \u201cEarth at Night\u201d). We wish to \nacknowledge their seminal contributions to showing and explaining images of Earth at \nnight for more than a decade.\nThank you to the following individuals from NASA\u2019s Science Support Office for pulling \nthis volume together:\nDouglas Bennett\nSally J. Bensusen\nHeather H.", + "\nThank you to the following individuals from NASA\u2019s Science Support Office for pulling \nthis volume together:\nDouglas Bennett\nSally J. Bensusen\nHeather H. Hanson\nMitchell K. Hobish\nWinn ie H. Humberson\nMarit A. Jentoft\u2013NilsenDeborah F . McLean\nKevin W. Miller\nAmy K. Mora n\nSteven M. Graham\nAlan B. Ward xv\nScience Content Reviewers\nMichael J. Falkowski \nJack A. Kaye\nMiguel O. Rom\u00e1n\nRanjay M. Shrestha\nEleanor C. Stokes\nSPECIAL THANKS\nLast, but certainly not least, we would like to thank: Thomas H. Zurbuchen, Associate \nAdministrator for NASA\u2019s Science Mission Directorate, for contributing the Foreword ; \nMichael H. Freilich, Former Director of NASA\u2019s Earth Science Division, for contributing \nthe Preface ; and James L. Green, NASA Chief Scientist, for contributing the Epilogue . \nWe would also like to thank Kristen J. Erickson, Emily M. Furfaro, Grey M. Hautaluoma, \nKirsten D. Petree, and Steven E. Platnick.2 Earth at NightAn Introduction to Nightlights\n 3\nDazzling photographs and images from space of our planet\u2019s nightlights have \n captivated public attention for decades.", + "2 Earth at NightAn Introduction to Nightlights\n 3\nDazzling photographs and images from space of our planet\u2019s nightlights have \n captivated public attention for decades. In such images, patterns are imme-\ndiately seen based on the presence or absence of light: a distinct coastline, bodies of water recognizable by their dark silhouettes, and the faint tendrils of roads and highways emanating from the brilliant blobs of light that are our modern, well-lit cities.\n4 Earth at Night\nBlue Marble Becomes Black Marble\n[Right] Dubbed the \u201cBlue \nMarble,\u201d this classic pho-tograph of Earth was taken by the Apollo 17 crew on December 7, 1972.\nThe color of an object is actually the color of the light reflected while all other colors \nare absorbed. Most of the light that is reflected by clear, open ocean water is blue, while \nthe red portion of sunlight is quickly absorbed near the surface. Therefore, very deep water with no reflections off the sea floor appears dark navy blue. Since approximately 70% of Earth\u2019s surface is covered by deep ocean water, Earth appears as a blue marble when viewed from space during daytime hours.", + " Since approximately 70% of Earth\u2019s surface is covered by deep ocean water, Earth appears as a blue marble when viewed from space during daytime hours. Clouds suspended in Earth\u2019s atmosphere provide the white swirls. 5\nEnshrouded in nighttime darkness, however, Earth appears more as a black marble \nfrom space\u2014one shimmering with light. The human search for light in darkness is long-\nstanding. Many of our myths and religions focus on this search, yet it is only within the past century that humans have gained the ability to take flight\u2014first to the skies and later into space\u2014providing new vantage points from which to view Earth and the twinkling lights below, clearly visible at night.\nFor nearly 25 years, satellite images of Earth at night have served as a fundamental \nresearch tool, while also stoking public curiosity. These images paint an expansive and revealing picture, showing how humans have illuminated and shaped the planet in profound ways since the invention of the light bulb 140 years ago.\nThese lights and the darkness tell stories about our planet\u2014stories that this volume \nwill present for your consideration.", + "\nThese lights and the darkness tell stories about our planet\u2014stories that this volume \nwill present for your consideration. Earth at Night will show how humans and natural phenomena light up the darkness, and how and why scientists have observed Earth\u2019s nightlights for more than four decades using both their own eyes and spaceborne instruments. It is an engaging and fascinating story; come along for the adventure!\n[Above] NASA\u2019s Black Marble. \nThese images of Earth at \nnight were created in 2012 over 9 days in April and 13 days in October. It took the Suomi National Polar-orbiting Partnership (NPP) satellite 312 orbits and 2.5 terabytes of data to get a clear shot of every parcel of Earth\u2019s land surface and islands. These new data were mapped over existing Blue Marble imagery to provide a realistic nighttime view of the planet. (See \u201cMaking a Cloud-Free, Global, Earth-at-Night Image Using NASA\u2019s Black Marble Product Suite\u201d in Appendix B \nfor technical \ndetails.)6 Earth at Night\nChapter TitleVision and Remote Sensing\nSeeing Is Sensing\nSeeing, or vision, is one of several senses we possess.", + ")6 Earth at Night\nChapter TitleVision and Remote Sensing\nSeeing Is Sensing\nSeeing, or vision, is one of several senses we possess. When you see something, \n your visual system absorbs light\u2014photons\u2014and your brain processes the \nview into something you can understand and, perhaps, act upon.\nHumans have long extended their visual capabilities with instruments\u2014such as \ntelescopes\u2014to enable them to see across distances that their own eyes could never \npossibly see. This is a form of \u201cremote sensing\u201d\u2014a capability that is at the core of our \nnew understanding of how Earth \u201cworks.\u201d\nOver time, science and technology developments have enabled us to create more-\nsensitive observing tools that far exceed the capabilities of relatively simple telescopes, \nproviding the ability to see things we had never even dreamed of previously. Our \ntechnologies can record phenomena well outside the realm of human visual abilities\u2014\nthings like heat and moisture, the measurement of which would have seemed magical \nto our forebears. But when it comes to inspiring the human spirit, there\u2019s nothing quite \nlike visible light, so light at ", + " But when it comes to inspiring the human spirit, there\u2019s nothing quite \nlike visible light, so light at night will be our focus here.\nWhat Is Light?\nUnless an object has a temperature of absolute zero (-273 \u00b0C) it reflects, absorbs, \nand emits energy\u2014called electromagnetic radiation\u2014in ways that depend on its \nphysical and chemical properties. The amount of electromagnetic radiation an object \nemits depends primarily on its temperature. The higher the temperature of an object, \nthe faster its electrons vibrate and the shorter the peak wavelength of the emitted 7radiation. Conversely, the lower the temperature of an object, the slower its electrons \nvibrate, and the longer its peak wavelength of emitted radiation. Given the wide range of temperatures in the universe, it should be clear that electromagnetic radiation has wavelengths that span a very wide range.\n[Above] Electromagnetic \nspectrum. The electromag-\nnetic spectrum is the range of energy waves of various wavelengths\u2014from the very short gamma and X-rays [right], through longer ultraviolet, visible, and infrared light, to ", + " The electromag-\nnetic spectrum is the range of energy waves of various wavelengths\u2014from the very short gamma and X-rays [right], through longer ultraviolet, visible, and infrared light, to microwaves and very long radio waves [left].\nThe electr\nomagnetic spectrum is the range of traveling waves of energy that run from \nvery short gamma and X-rays through ultraviolet light, visible light, microwaves, and out \nto long radio waves. Visible light is the quite narrow band that human eyes are adapted to see. But as noted earlier, humans are proficient at developing tools to extend the range of their capabilities. Using sensors on orbiting satellites, designed to detect multiple spectral-band combinations, scientists can \u201ctune in\u201d to study various aspects of Earth\u2019s surface in ways not possible from a simple color photograph. Just as soils, 8 Earth at Nightdifferent plant types, plant health, the presence of water, bare rock, ice, and many other \ntypes of land cover each have unique \u201csignatures\u201d in the electromagnetic spectrum, different sources of light (e.g., incandescent lamps and LEDs) have their own unique signatures in the electromagnetic spectrum.", + "plant types, plant health, the presence of water, bare rock, ice, and many other \ntypes of land cover each have unique \u201csignatures\u201d in the electromagnetic spectrum, different sources of light (e.g., incandescent lamps and LEDs) have their own unique signatures in the electromagnetic spectrum. Over time, scientists can observe and analyze changes in spectral signatures to detect changes in Earth\u2019s surface and identify various phenomena taking place (including those dealing with water, vegetation, soil, and the like\u2014see the \u201cSpectral signatures\u201d figure below), sometimes based on lighting patterns.\nRemote Sensing\nPlacing vision-augmenting tools\u2014sensors\u2014on satellites that orbit Earth has \nexpanded our ability to see our home planet in a new light.\nObserving Earth from space using sensors on satellites (i.e., remote sensing) traces its \norigins to the early days of the space age\u2014both Russian and American programs\u2014when surface-imaging sensors first flew onboard aircraft and then later on spacecraft. With the emergence of manned space flight in the 1960s, Earth-orbiting cosmonauts and astro-nauts acted much like tourists by taking photos from the windows of their spacecraft.\nCredit: U.", + " With the emergence of manned space flight in the 1960s, Earth-orbiting cosmonauts and astro-nauts acted much like tourists by taking photos from the windows of their spacecraft.\nCredit: U.S. Geological Survey (USGS)/NASA[Below] Spectral signatures. \nSatellite instruments measure \nlight emitted or reflected back to space at different wave-lengths and create spectral reflectance curves. Differ -\nences in the shape of the \u201cspectra\u201d can be used to determine what the satel-lite is \u201cseeing,\u201d such as soil, green vegetation, and water. 9\nRemote sensing is more than simply taking pictures, however. It more fully \ndescribes the science\u2014and art\u2014of observing, identifying, and measuring an \nobject without coming into direct contact with it. This process involves detecting and measuring radiation of different wavelengths emitted or reflected from distant objects or materials. Emitted light is comprised of photons in an excited state; for example, light coming from the Sun or a light bulb filament. Reflected light is light from another source bounced off an object\u2019s surface.\n10 kmCredit: USGS/NASAcolder warmer\nThermal SignatureLarsen C \nIce ShelfA-68AA-68B\nsea ice\nJuly 21, 2017[Left]", + "\n10 kmCredit: USGS/NASAcolder warmer\nThermal SignatureLarsen C \nIce ShelfA-68AA-68B\nsea ice\nJuly 21, 2017[Left] Seeing ice at night. \nA massive iceberg (A-68) first broke away from \nAntarctica\u2019s Larsen C ice shelf sometime between July 10 and July 12, 2017. The fractured berg (A-68A and A-68B) and shelf are visible in this image, acquired on July 21, 2017, by the Thermal Infrared Sensor (TIRS) on the Landsat 8 satellite. This false-color view shows the relative warmth or coolness across the region. White indicates where the ice or water surface is warmest; dark grays and blacks are the coldest areas of ice.10 Earth at NightOrbiting Tools to Observe Nightlights\n 11\nThe original motivation for making low-light Earth observations was practical: military meteorologists wanted \nto track clouds and storms, smoke plumes, and dust storms at all hours, to better support military planning \nand execution. The Air Force, in particular, needed to provide more accurate and timely forecasts for pilots working at night over both land and sea.", + " The Air Force, in particular, needed to provide more accurate and timely forecasts for pilots working at night over both land and sea.\n12 Earth at Night\n[Previous pages] Vintage \nphotos of Earth at night\u2014\nNASA\u2019s Mercury-Atlas \nmission. The photograph \non page 10 was taken \nfrom NASA\u2019s unmanned Mercury-Atlas 4 mission on \nSeptember 13, 1961. High \nclouds (white) are mixed with Earth\u2019s nightlights (yel-\nlow) on the right-hand side \nof the image, with Earth\u2019s \nhorizon on the left-hand \nside. The handwritten annotation at the top (that \nappears backwards in that \nphoto) indicates this was the 348\nth photograph taken \non the MA-4 mission. The \nnext photograph ( page 11) \ntaken by MA-4, number \n349, depicts an even darker \nscene of Earth at night. The white line in this photo is \ndue to a film artifact.So, scientists have observed Earth\u2019s nightlights for more than four decades, first \nwith astronaut photography and military satellites. Since the 1960s, the U.S. Air \nForce has operated the Defense Meteorological Satellite Program (DMSP), a series of 18 polar-orbiting satellites that observe clouds and other weather ", + " Air \nForce has operated the Defense Meteorological Satellite Program (DMSP), a series of 18 polar-orbiting satellites that observe clouds and other weather variables in key wavelengths of infrared and visible light. Starting in 1972, the DMSP satellites included the Operational Linescan System (OLS), which gives weather forecasters some ability to see in the dark.\nWhile DMSP has been a source of nighttime images for decades, until fairly recently \nthe data were classified, which meant that only a few civilian scientists could conveniently gain access to study the data. The atmospheric science community was eager to have a more accessible night-vision tool to better understand weather and climate patterns and phenomena. Finally, in 2011, a new source of unclassified satellite images of Earth at night became available\u2014one that improved upon the capabilities of OLS. The new low-light sensor was called the Visible Infrared Imaging Radiometer Suite (VIIRS), and was launched in October 2011 onboard the Suomi National Polar-orbiting Partnership (NPP) satellite\u2014a partnership between NASA, the National Oceanic and Atmospheric Administration (NOAA)", + "(VIIRS), and was launched in October 2011 onboard the Suomi National Polar-orbiting Partnership (NPP) satellite\u2014a partnership between NASA, the National Oceanic and Atmospheric Administration (NOAA), and the U.S. Department of Defense.\nVIIRS currently flies on Suomi NPP as well as on the NOAA-20 satellite (launched \nNovember 2017). The VIIRS \u201cDay/Night Band,\u201d or DNB, can observe dim light down to the scale of an isolated highway lamp or fishing boat. It can even detect faint, nocturnal atmospheric light\u2014known as airglow\u2014and observe clouds lit by it. Through the use of its DNB, VIIRS has made the first quantitative measurements of light emissions and reflections from Earth\u2019s surface, distinguishing the intensity and the sources of nighttime light. The combination of these measurements gives us a global view of the human and natural light sources on Earth. Looking ahead, three more satellite missions are planned over the next decade, with a VIIRS instrument scheduled to be onboard each platform.\nThanks to advancements in sensor technology and improved optics, the VIIRS \nDNB is ten-to-fifteen times better than the OLS sensor at resolving the relatively dim 13\n1980s\n[Above] Early ", + "\nThanks to advancements in sensor technology and improved optics, the VIIRS \nDNB is ten-to-fifteen times better than the OLS sensor at resolving the relatively dim 13\n1980s\n[Above] Early map of Earth at night. Technology has come a long way since the 1980s and 1990s. This map shows \nthe visible light sour\nces on Earth\u2019s surface as measured by the OLS on the DMSP satellites during the 1980s. White dots \nr\nepresent city lights, red dots represent forest fires, and yellow dots represent oil-well fires.\n2016\n[Above] The 2016 Black Marble. The 2016 Earth at night image, dubbed NASA\u2019s Black Marble , was created with \nVIIRS DNB data fr\nom the Suomi NPP satellite. The data were acquired over 190 days in 2016. This translates into \n5247 orbits and 42 terabytes of input data to get a cloud-fr\nee composite image of Earth\u2019s land surfaces and ocean.14 Earth at Nightlights of human settlements and reflected moonlight. Just like the picture on your television set, satellite images \nare made up of tiny squares. These squares are called pixels\u2014short for picture elements\u2014and represent the amount of reflected or emitted light energy recorded for that part of the image.", + " These squares are called pixels\u2014short for picture elements\u2014and represent the amount of reflected or emitted light energy recorded for that part of the image. Each VIIRS pixel covers a distance of roughly 0.46 miles (742 meters) across, compared to the 1.86-mile (3-kilometer) footprint of OLS. To learn more about the technical details of OLS and VIIRS, see Appendix C.\nOLS\nDelhi\nNovember 11, 2012\n100 km\nVIIRS\nNovember 12, 2012\nDelhi\n100 km[Right] This image pair illustrates the differences in \nresolution between OLS and VIIRS. On November 11, 2012, the OLS on a DMSP spacecraft captured the top image of city, village, and highway lights near Delhi, India. For comparison, the VIIRS instrument on the Suomi NPP satellite acquired the bottom image showing this same area one night later.\nAs they have done since the dawn of the space age, astronauts onboard the International Space Station\n \n(ISS) also take photographs of Earth at night. In fact, their images have even higher spatial resolution than \nimages from VIIRS (typically tens-to-hundreds of meters per pixel).", + " In fact, their images have even higher spatial resolution than \nimages from VIIRS (typically tens-to-hundreds of meters per pixel). However, unlike satellites, astronauts are not looking at Earth 24 hours a day, 7 days a week, 365 days a year. Due to the orbit of the ISS, astronauts 15\ncannot see the surface in the polar regions, although they can observe phenomena in the upper atmosphere. \nAlso, the ISS passes over a given point on Earth every two or three days and at variable times, while, for example, Suomi NPP flies over the same point twice a day at roughly the same time each day. With their complementary capabilities of spatial resolution, timing, and quantitative accuracy, scientists are interested in making joint observations with VIIRS and the ISS.\n2003 OLS\n100 km\n2012 VIIRS\n100 km\n2016 VIIRS\n100 km\nJane Addams \nMemorial \nTollway\n[Above] Change Over Time\u2014Chicago, Illinois. These images show the area surrounding Chicago, Illinois, using nightlight \ndata from the OLS onboard the DMSP released in 2003 (left), the VIIRS DNB in 2012 (middle), and the VIIRS DNB in 2016 (right).", + "images show the area surrounding Chicago, Illinois, using nightlight \ndata from the OLS onboard the DMSP released in 2003 (left), the VIIRS DNB in 2012 (middle), and the VIIRS DNB in 2016 (right). \nWhen you look at the images side by side, the OLS image appears coarse and blurry compared to the high-precision VIIRS images from 2012 and 2016. The most readily noticeable difference in these nighttime composite views of Chicago and surrounding areas in 2012 and 2016 is lighting along a recently expanded section of Interstate 90. This part of the high -\nway, the Jane Addams Memorial Tollway, links Chicago with Rockford, Illinois, to the northwest.\nWhile scientists use multiple spectral band combinations covering differ\nent parts of the electromagnetic \nspectrum\u2014such as visible and infrared\u2014during nighttime hours, this book focuses mainly on the detection \nof visible light at night. For comparison, some daytime visible and infrared images will be shown alongside some of the nighttime visible images.16 Earth at NightImprovements in ISS Photography\u2014Northeastern United States\nThis pair of photographs, centered on New York City in the northeastern United States, \nshows improvements in ", + "16 Earth at NightImprovements in ISS Photography\u2014Northeastern United States\nThis pair of photographs, centered on New York City in the northeastern United States, \nshows improvements in camera technology used on the ISS over a 14-year time period. \nThe image on the left was taken on January 18, 2003, with a Nikon D1 digital camera (3 megapixels), while the image on the right was take on January 10, 2017, with a Nikon D4 digital camera (16 megapixels).\nJanuary 18, 2003\nNew Haven\nBridgeport\nLong Island Sound\nNew New \nYorkYork\nCityCity\nNewarkAppalachian \nMountains\nPhiladelphiaAtlantic Ocean\n 17The 2003 image was taken from a vantage point well to the northeast of the cities, \nwith the camera pointed westward back towards New York City and the coast. The result \nis that the perspective is distorted but still recognizable. Low clouds have formed over the waters of the Atlantic and have settled into some of the valleys of the Appalachian Mountains to the northwest.\nJanuary 10, 2017\nHartford\nNew Haven\nBridgeport\nLong Island Sound\nNew \nYork\nCity\nNewark\nAtlantic Ocean\nbarrier island\n18 Earth at NightComing into Focus\u2014Cairo, Egypt\nThis pair of photographs, centered on Cairo, ", + "\nJanuary 10, 2017\nHartford\nNew Haven\nBridgeport\nLong Island Sound\nNew \nYork\nCity\nNewark\nAtlantic Ocean\nbarrier island\n18 Earth at NightComing into Focus\u2014Cairo, Egypt\nThis pair of photographs, centered on Cairo, Egypt, shows improvements in camera \ntechnology used on the ISS between October 7, 2003 (left) and December 10, 2014 \n(right). The 2003 photo was taken with a Kodak DCS 760c electronic still camera \n(6.3 megapixels), while the 2014 image was taken with a Nikon D4 digital camera \n(16 megapixels).\nOctober 7, 2003\nCairo\nNile River 19\nDecember 10, 2014\nCairoCairo\nNile River\nNew Cairo\n El Shorok\nEl AborNile Delta\nSixth of October\nCity\n20 Earth at NightWhere Do Nightlights Come From?\nThe night side of Earth twinkles with light, easily observed by \norbiting satellites. The most obvious source comes from city \nlights, but as one looks closer, much more becomes discernible. Even at some distance from human settlements, light still shines: Wildfires and volcanoes rage; gas flares glow like candles; auroras dance across the polar skies; moonlight and starlight reflect off surface water, snow, and deserts; and clouds are easily seen.", + "shines: Wildfires and volcanoes rage; gas flares glow like candles; auroras dance across the polar skies; moonlight and starlight reflect off surface water, snow, and deserts; and clouds are easily seen. Even the air and ocean some-times glow.\nThe \u201cNature\u2019s Light Shows\u201d and \u201cHuman Light Sources\u201d sections of \nthis book feature images of natural and anthropogenic light sources to illustrate how scientists use nightlight data to study our changing planet and how decision makers, in turn, can use the knowledge gained for public benefit. Some of these applications include forecasting a city\u2019s energy use and carbon emissions, eradicating energy poverty and fostering sustainable energy development, providing immediate information when disasters strike, and monitoring the effects of conflict and population displacement.\n[Right] An astronaut onboard the International Space Station took this nighttime \npanorama while looking north across Pakistan\u2019s Indus River valley. The port city of Karachi is the bright cluster of lights facing the Arabian Sea, which appears completely black. City lights and the dark color of dense agriculture closely track with the great curves of the Indus valley.", + " City lights and the dark color of dense agriculture closely track with the great curves of the Indus valley. For scale, the distance from Karachi to the foothills of the Himalaya Mountains is 720 miles (1,160 kilometers). This photograph shows one of the few places on Earth where an international boundary can be seen at night. The winding border between Pakistan and India is lit by security lights that have a distinct orange tone.\nSeptember 23, 2015\nPakistanAfghanistan\nKarachi 21\nIndia\nborderIndus River\nValleyHimalaya Mountains\nairglow\n22 Earth at NightBeyond City Lights\u2014Java Sea\nThe visible lights produced by modern civilization are often obscured \nby the more intense light of the Sun during the day. However, as this \nphotograph\u2014taken over the Java Sea by a member of the Expedition 42 crew onboard the ISS\u2014reveals, at night these \u201cnighttime lights\u201d become visible. To the discerning eye, these lights reveal intricate details of both natural phenomena and the unmistakable footprints of human civilization.", + " To the discerning eye, these lights reveal intricate details of both natural phenomena and the unmistakable footprints of human civilization. The photo, taken on October 20, 2014, captured light from roads and cities, fishing boats, lightning, airglow, and even a few gas flares. North of the city of Surabaya, the capital of East Java province, large numbers of boat lights dot the darkened Java Sea. Many of these vessels are likely fishing boats using bright lights to attract squid and other sea life. According to one estimate based on satellite observations, there were over 500 fishing boats in the area on a single night in September 2014. The same study pointed out that two of the brighter points of light in the area are gas flares emanating from offshore oil infrastructure. In addition to the human activity, the photograph also showcases a few natural splashes of light and color. The blue-white patch on the far left is a flash of lightning, a common sight for astronauts. The glowing line along Earth\u2019s limb\u2014or edge of the atmosphere\u2014is airglow, a type of light that is produced by chemical reactions in the upper atmosphere.", + " The glowing line along Earth\u2019s limb\u2014or edge of the atmosphere\u2014is airglow, a type of light that is produced by chemical reactions in the upper atmosphere. While airglow can also be green or blue, red airglow like this comes from excited oxygen atoms at heights of about 90 to 190 miles (150 to 300 kilometers).\nOctober 20, 2014\nJava Sealightning\n 23\nairglow\nboat lights\nSurabayagas flare24 Earth at NightMoonlight, Old Lights, and New Lights\u2014Gulf of Mexico\nThis wide-angle, nighttime photograph was taken on February 11, 2015, by ISS \nastronauts looking southeastward over the Gulf of Mexico. Moonlight reflects diffusely \noff the waters of the Gulf, making the largest illuminated area in the image. The sharp edges of light patterns from coastal cities trace the long curve of the shoreline from \nFebruary 11, 2015\nmoonglint\nHouston\nAustin\nSan Antonioshale fracking\nlightsCorpus Christi\nWaco\nNew Orleans\n 25\nNew Orleans, at the mouth of the Mississippi River, to Brownsville, Texas, in the \nwesternmost Gulf. In recent years a new pattern of lights has appeared on the land-scape and revealed the oil- and gas-production zone of south-central Texas.", + " In recent years a new pattern of lights has appeared on the land-scape and revealed the oil- and gas-production zone of south-central Texas. This long, less-dense swath of pinpoints spreads across 210 miles (330 kilometers) of what is now known as shale-fracking country.\nGulf of Mexico\nLaredoBrownsville\nMexico City\nMonterrey\n26 Earth at Night\nA World of Change\nThis map shows the change in lighting intensity from 2012 to 2016. The map was created using \ntwo separate nightlight datasets (from 2012 and 2016) derived using data from the VIIRS instrument \non Suomi NPP . Dark purple represents areas with new light since 2012, while dark orange represents \nChange in Nightlights (2012\u20132016)\nLights out No c hange New lights 27\nareas where light existed in 2012 but no longer exists in 2016. Areas where lighting intensity stayed \nthe same between 2012 and 2016 appear white. Varying shades of purple and orange indicate \nareas that have become brighter or dimmer since 2012, respectively. To view and learn about a few \nregional examples up close, see stories on pages \n114\u2013115 and 140\u2013141.", + " To view and learn about a few \nregional examples up close, see stories on pages \n114\u2013115 and 140\u2013141.\n28 Earth at NightNature\u2019s Light Shows\nAt night, with the light of the Sun removed, nature\u2019s brilliant \n glow from Earth\u2019s surface becomes visible to the naked \neye from space. Some of Earth\u2019s most spectacular light shows are \nnatural, like the aurora borealis, or Northern Lights, in the Northern Hemisphere (aurora australis, or Southern Lights, in the Southern Hemisphere). The auroras are natural electrical phenomena caused by charged particles that race from the Sun toward Earth, inducing \nchemical reactions in the upper atmosphere and creating the appearance of streamers of reddish or greenish light in the sky, usually near the northern or southern magnetic pole. Other natural lights can indicate danger, like a raging forest fire encroaching on a city, town, or community, or lava spewing from an erupting volcano.\nWhatever the source, the ability of humans to monitor nature\u2019s \nlight shows at night has practical applications for society.", + "\nWhatever the source, the ability of humans to monitor nature\u2019s \nlight shows at night has practical applications for society. For example, tracking fires during nighttime hours allows for continuous \nmonitoring and enhances our ability to protect humans and other animals, plants, and infrastructure. Combined with other data sources, our ability to observe the light of fires at night allows emergency managers to more efficiently and accurately issue warnings and evacuation orders and allows firefighting efforts to continue through the night. With enough moonlight (e.g., full-Moon phase), it\u2019s even possible to track the movement of smoke plumes at night, which can impact air quality, regardless of time of day.\nAnother natural source of light at night is emitted from glow-\ning lava flows at the site of active volcanoes. Again, with enough \nSeptember 17, 2011 29\n[Above] Astronauts onboard the International Space Station are treated to a spectacular display of the aurora australis while in orbit \nover Southern New Zealand on September 17, 2011.", + " Again, with enough \nSeptember 17, 2011 29\n[Above] Astronauts onboard the International Space Station are treated to a spectacular display of the aurora australis while in orbit \nover Southern New Zealand on September 17, 2011.30 Earth at Nightmoonlight, we can even track volcanic ash plumes, which can be hazardous to airplanes \nin flight. The ash, made up of tiny pieces of glass and rock, is abrasive to engine turbine blades and can melt on the blades and other engine parts, causing damage and even engine stalls, with consequent danger to the plane\u2019s integrity and passenger safety. Vol -\ncanic ash also reduces visibility for pilots and can cause etching of windshields, further reducing pilots\u2019 ability to see. Nightlight images can be combined with thermal images to give a more complete view of volcanic activity on Earth\u2019s surface, as will be seen later.\nThe VIIRS DNB takes advantage of moonlight, airglow (the atmosphere\u2019s self-illu-\nmination through chemical reactions), zodiacal light (sunlight scattered by interplanetary dust), and starlight from the Milky Way. By using these dim light sources, the DNB can detect changes in clouds, snow cover, and sea ice.", + ", zodiacal light (sunlight scattered by interplanetary dust), and starlight from the Milky Way. By using these dim light sources, the DNB can detect changes in clouds, snow cover, and sea ice. Geostationary Operational Environmental Satellites (GOES), managed by the National Oceanic and Atmospheric Administration (NOAA), orbit over Earth\u2019s equator, offering uninterrupted observations of North America, whereas high-latitude areas such as Alaska benefit from polar-orbiting satellites like Suomi NPP . Polar-orbiting satellites provide significant overlapping coverage at the poles, enabling more data to be acquired in these regions. For exam-ple, during polar darkness (i.e., winter months), VIIRS DNB data allow scientists to observe sea ice formation and snow cover extent at the highest latitudes, in addition to spotting where there is clear water for ships to pass through.\nThe use of nightlight data by weather forecasters is growing as the VIIRS instrument can \nbe used to see clouds at night when they are lit by moonlight and lightning. Scientists use nightlight data to study nighttime behavior of weather systems, including severe storms, which can develop and strike populous ", + " Scientists use nightlight data to study nighttime behavior of weather systems, including severe storms, which can develop and strike populous areas at night as well as during the day. Combined with thermal data, visible nightlight data can also be used to see clouds at various heights in the atmosphere at night, such as dense marine fog, which allows weather forecasters to issue marine advisories with higher confidence than ever before and, \ntherefore, greater utility\u2014see \u201cMarine Layer Clouds\u2014California\u201d on \npage 56.\nIn this section of the book you will see how nightlight data are used to observe nature\u2019s \nspectacular light shows across a wide range of sources.[Opposite page] For the first \ntime in perhaps a decade, Mount Etna experienced a \u201cflank eruption\u201d\u2014erupting from its side instead of its summit\u2014on December 24, 2018. The activity was accompanied by 130 earth -\nquakes occurring over three hours that morning. Mount Etna, Europe\u2019s most active volcano, has seen periodic activity on this part of the mountain since 2013. The Operational Land Imager (OLI) on the Landsat 8 satellite acquired the main image of Mount Etna on ", + " The Operational Land Imager (OLI) on the Landsat 8 satellite acquired the main image of Mount Etna on December 28, 2018.The inset image highlights the active vent and thermal infrared signature from lava flows, which can be seen near the newly formed \nfissure on the southeastern side of the volcano. The inset was created with data from OLI and the Thermal Infrared Sensor (TIRS) on Landsat 8. Ash spewing from the fissure cloaked adjacent villages and delayed aircraft from landing at the nearby Catania airport. Earth-quakes occurred in the subsequent days after the initial eruption and displaced hundreds of people from \ntheir homes. For nighttime \nimages of Mt. Etna\u2019s March 2017 eruption, see \npages 48-51. 31\nMount\nEtna\nash\nDecember 28, 2018\nthermal infrared\nsignature\nactive \nvent\nash\nCredit: USGS/NASA32 Earth at NightFiresFires\nTracking Fires Day and Night\u2014Northwest United States\nIn summer 2015, wildfires raged across the western contiguous United States and \nAlaska. Many of those fires burned in the Northwest United States, visible in these \nimages from late August.", + " Many of those fires burned in the Northwest United States, visible in these \nimages from late August. The nighttime image (top, opposite) was acquired in the early morning (local time) on August 19, 2015, by the VIIRS DNB on the Suomi NPP satellite. Labels point to the large, actively burning fires in the region.\nThe daytime image (bottom, opposite) shows the same area in natural color, acquired \nduring the afternoon on August 18, 2015, with the Moderate Resolution Imaging Spectroradiometer (MODIS) on NASA\u2019s Aqua satellite. Red outlines indicate hot spots where MODIS detected unusually warm surface temperatures, generally associated with fires. Thick plumes of smoke are visible emanating from the hot spots.\nAccording to the Northwest Interagency Coordination Center, the Okanogan Complex \nFire in Washington was among the larger active fires; as of August 20, the fire had burned 91,314 acres (143 square miles, or 370 square kilometers). In Oregon, the Canyon Creek Complex Fire had burned 48,201 acres (75 square miles, or 195 square kilometers), destroyed 26 residences, and threatened ", + " In Oregon, the Canyon Creek Complex Fire had burned 48,201 acres (75 square miles, or 195 square kilometers), destroyed 26 residences, and threatened another 500. Both fires were less than 40 percent contained at the time the images were acquired. 33Fires\nAugust 19, 2015\nOregonWashington\nIdahoMontana\nWyoming\nPortland\nBoise\nSeattle\nSpokane\nRapid Wildfire\nCampbell Wildfire\nNorth Star\n Okanogan Complex\nCanyon Creek\nNational Creek ComplexCougar CreekBlack Canyon\n100 km100 km\nAugust 18, 2015\nWashington\nOregon\nIdahoMontana\nWyoming\nNational Creek Complex\nCougar Creek\nNorth Star\nOkanogan Complex\nCanyon Creek\nCampbell WildfireCampbell Wildfire\nRapid Wildfire\nBlack Canyon\n100 km100 km\n34 Earth at NightFires\nSeptember 1, 2013\nAugust 20, 2013\nRim Fire\nYosemite21 22 23\n24 \nTuolumne \nCity\n25 26 27\n28 29 30 31\n2 3 4\n50 km\n 35Fires\nProgression of the Rim Fire\u2014Yosemite, California\nThe winter of 2012\u20132013 was among the driest on record for California, setting \nthe stage for an active fire season in the summer of 2013. At the time, the Rim Fire \nwas the third largest in California since record-keeping began in 1932.", + " At the time, the Rim Fire \nwas the third largest in California since record-keeping began in 1932.\nThe VIIRS DNB on the Suomi NPP satellite tracked the growth of the fire between \nAugust 20 and September 4, 2013. The brightest, most intense parts of the fire glow white. Pale gray smoke streams away, generally to the north. Thin clouds obscured the view on some days, particularly August 31, September 1, and September 3. On August 20, the Moon was full, so the landscape reflected a large amount of moonlight. The background grew progressively darker as the new Moon approached on September 5.\nThe perimeter of the fire changed along different fronts from day to day, depending \non winds and firefighting efforts. On August 24, firefighters focused on containing the western edge of the fire to prevent it from burning into Tuolumne City and the populated Highway 108 corridor. They also fought the eastern edge of the fire to protect Yosemite National Park (outlined in yellow). These efforts are evident in the images: Between August 23 and 24, the eastern edge held steady, the western edge ", + " These efforts are evident in the images: Between August 23 and 24, the eastern edge held steady, the western edge receded, and the fire grew in the southeast.\nOn the morning of August 25, 2013, fire managers reported that the blaze was \ngrowing in the north and east. With the fire burning aggressively and moving east into Yosemite, August 26 and 27 proved challenging days for firefighters. But over the next few days, they began to gain control after a series of burnout operations along the fire\u2019s northern and eastern edges. On August 29, the evacuation advisory for Tuolumne City was lifted. The southeastern flank continued to burn intensely into the first week of September.36 Earth at NightFires\nSpreading Like Wildfire\u2014Yakutia, Siberia, Russia\nOn August 3, 2012, the MODIS instrument on NASA\u2019s Aqua satellite acquired a \ndaytime view of a group of wildfires blazing in eastern Siberia, as shown in the top \npanel. The fires detailed in the white box were burning in the southeastern part of Yakutia, near the port city of Yakutsk along the Lena River.", + " The fires detailed in the white box were burning in the southeastern part of Yakutia, near the port city of Yakutsk along the Lena River. Red dots indicate hot spots where MODIS detected unusually warm surface temperatures associated with fires. Smoke is visible emanating from the hot spots.\nThe VIIRS DNB on Suomi NPP acquired a nighttime image of that same group of \nfires on the same date, as seen in the bottom panel. In this image, the brightest fires are white; smoke is light gray.\nThe vast majority of Russian wildfires occur in Siberia, generally along the southern \nborder with China and Mongolia. 37Fires\nAugust 3, 2012August 3, 2012\nSea of Okhotsk\n100 km\n50 km\nclouds\nsmoke\nfires\nAugust 3, 201238 Earth at NightFires\nApril and October 2012\nAustralia\nPerth\nAdelaide\nMelbourneSydneyBrisbane\nfires\n500 km\nOctober 11\u201324, 2012\nAustralia\nPerth\nAdelaide\nMelbourneSydneyBrisbaneburn scars\n500 kmN 39Fires\nCity Lights, or Not\u2014Australia\nThe top image at left shows the nightlights of Australia as observed by the VIIRS \nDNB on the Suomi NPP satellite in April and October 2012.", + "scars\n500 kmN 39Fires\nCity Lights, or Not\u2014Australia\nThe top image at left shows the nightlights of Australia as observed by the VIIRS \nDNB on the Suomi NPP satellite in April and October 2012. The composite image \nincludes manmade light sources and the light from wildfires.\nThe lower map is a mosaic showing the burned areas of the landscape (red) from \nOctober 11\u201324, 2012, combined with urban areas (black). The data were collected by the MODIS instruments on NASA\u2019s Terra and Aqua satellites. In effect, the lower map shows where fires burned that month. Though many rural areas of interior Australia are dry and relatively barren by some standards, there is still enough vegetation to burn. Not every light in the night view matches up with a fire\u2014partly because the fire map does not include fires from April and partly because not every fire leaves a scar that is detectable from space. Even simple cloud cover could prevent burn scars from being observed. Aside from the fires, some of the nightlights appearing in uninhabited areas can be attributed to lightning, flares of natural gas and other activities from oil drilling or mining operations, and fishing boats\u2014all of which can ", + " Aside from the fires, some of the nightlights appearing in uninhabited areas can be attributed to lightning, flares of natural gas and other activities from oil drilling or mining operations, and fishing boats\u2014all of which can show up as points of light.40 Earth at NightFiresBillowing Smoke in the Night \u2014Idaho and Montana\nAugust 29, 2012\nMontana\nIdahoOregonWashington\nBoiseHalstead Fire\nTrinity Ridge Fire\nMustang Complex Fireclouds\n50 km\nOn August 29, 2012, the VIIRS DNB on the Suomi NPP satellite captured this nighttime view of wildfires burning \nin Idaho and Montana. When the image was acquired, the Moon was in its waxing gibbous phase (i.e., it was \nmore than half lit but less than full). Numerous hot spots from the Mustang Complex Fire are visible in northern 41Fires\nIdaho. A plume of thick, billowing smoke streams west from the brightest fires near the \nIdaho\u2013Montana border. The Halstead and Trinity Ridge fires are visible to the south. In \naddition to the fires, city lights from Boise and other smaller cities appear throughout the image. A bank of clouds is located west of the Mustang Complex, over southeastern Washington and northeastern Oregon.", + " A bank of clouds is located west of the Mustang Complex, over southeastern Washington and northeastern Oregon.\nMustang Complex Firesmoke\n20 km\n42 Earth at NightFiresFires Light Up Mount Vesuvius\u2014Naples, Italy\nThe forest preserve on Mount Vesuvius normally keeps the Italian mountain \nshrouded in darkness when viewed in nighttime photographs by astronauts or in \nsatellite images of Naples. However, that was not the case on July 12, 2017, when the VIIRS DNB on Suomi NPP captured an image (below, right) of wildfires lighting up the slopes of the volcano. For comparison, the July 9, 2017, image (below, left) was taken before the fires began. The July fires burned much of the woodlands in Vesuvius National Park, which was established in 1995. The park protects more than 600 \nJuly 9, 2017\n50 km\nMt. Vesuvius\nNaples\nJuly 12, 2017\nMt. Vesuvius\nNaples\n 43Fires\ntypes of plants, 100 species of birds, and many small mammals and reptiles. Mount \nVesuvius is best known for a catastrophic eruption in 79 A.D. that destroyed the cities of Pompeii and Herculaneum.", + " Mount \nVesuvius is best known for a catastrophic eruption in 79 A.D. that destroyed the cities of Pompeii and Herculaneum. Its last major eruption occurred in 1944.\nFor comparison, the MODIS instrument captured this daylight image (below) of \nthe fires (red dots) on July 12. Note how the extent of Naples and the area around the volcano are less clearly defined.\nJuly 12, 2017\nMt. Vesuvius\nNaples\n50 km\n44 Earth at NightVolcanoesVolcanoes\nThe Infrared Glows of Kilauea\u2019s Lava Flows\u2014Hawaii\nIn early May 2018, an eruption on Hawaii\u2019s Kilauea volcano began to unfold. The \neruption took a dangerous turn on May 3, 2018, when new fissures opened in the resi-\ndential neighborhood of Leilani Estates. During the summer-long eruptive event, other fissures emerged along the East Rift Zone. Lava from vents along the rift zone flowed downslope, reaching the ocean in several areas, and filling in Kapoho Bay.\nA time series of Landsat 8 imagery shows the progression of the lava flows from \nMay 16 to August 13. The night view combines thermal, shortwave infrared, and near-infrared wavelengths to tease out the very hot lava (", + " The night view combines thermal, shortwave infrared, and near-infrared wavelengths to tease out the very hot lava (bright white), cooling lava (red), and lava flows obstructed by clouds (purple). 45Volcanoes\nMay 16, 2018\nVacationland\nHawaiiLeilani EstatesKapoho \nBay\nMay 23, 2018\n3 km\nJune 8, 2018\nKapoho \nBay\nJune 10, 2018\nJuly 3, 2018\n July 12, 2018\nJuly 19, 2018\n August 13, 201846 Earth at NightVolcanoesMount Tongariro Erupts\u2014New Zealand\nLate on August 6, 2012, New Zealand\u2019s Mount Tongariro erupted for the first time \nin 115 years, spewing a cloud of ash over North Island, closing roads, and cancelling \ndomestic flights. The image at right, observed by the VIIRS DNB on Suomi NPP , shows Mount Tongariro as it appeared at 12:55 a.m. New Zealand time on August 7.\nThe volcano lies in a mostly undeveloped part of the country. The 6,490-foot \n(1,978-meter) peak is a popular spot for hikers and is right next to Mount Ngauruhoe, the stand-in for Mount Doom in the \u201cLord of the Rings\u201d movies. Both Tongariro and Ngauruhoe are part of the circum-Pacific seismic belt or \u201cRing of Fire\u201d region, known for its ", + " Both Tongariro and Ngauruhoe are part of the circum-Pacific seismic belt or \u201cRing of Fire\u201d region, known for its seismic activity and volcanism.\nTongariro sent ash at least 20,000 feet (6,100 meters) into the air. Because of \nthe impact to aircraft operations, the New Zealand Civil Aviation Authority issued advisories to reroute air traffic around the ash plume so as to avoid damage to aircraft engines as happened in 1989 when a KLM Boeing 747 commercial jet flew into an ash plume produced by Alaska\u2019s Mt. Redoubt, which resulted in damage to all four engines. Swift, skillful piloting kept this incident from being a tragedy, as all engines were subsequently restarted, and the aircraft landed safely.\nThe first appearance of the ash plume occurred far downwind of Tongariro in thermal \nimagery, as ash was lofted high enough to produce a strong thermal contrast in the atmosphere. Images from the VIIRS DNB can be combined with thermal images such as from VIIRS (not shown here) to give a more-complete view of such activity on Earth\u2019s surface.", + " Images from the VIIRS DNB can be combined with thermal images such as from VIIRS (not shown here) to give a more-complete view of such activity on Earth\u2019s surface. Getting thermal data closer to a volcano with conventional infrared sensors is difficult, but VIIRS can detect such signals close to the source and lower in the atmosphere than other sensors. 47Volcanoes\nAugust 7, 2012\nTongariro\nclouds\nash plume\nLake Taupo\nTauranga\n50 km\nNapier\n48 Earth at NightVolcanoes\nNighttime Glow at Mount Etna\u2014Italy\nAt about 2:30 a.m. local time on March 16, 2017, the VIIRS DNB on the Suomi NPP \nsatellite captured this nighttime image of lava flowing on Mount Etna in Sicily, Italy. \nEtna is one of the world\u2019s most active volcanoes. 49Volcanoes\nMarch 16, 2017\nSicily\nhot lava\nCatania\nMediterranean Sea\nPalermo\nMarsala\nMalta\nMount Etna\n50 km\n50 Earth at NightVolcanoesMount Etna Erupts\u2014Italy\nMarch 19, 2017\nMt. Etna\n(southeastern cone)\nlateral vent\nResort\nBiancavilla\nGiarre\nAcireale\nThe highly active Mount Etna in Italy sent red lava rolling down its flank on \nMarch 19, 2017.", + " Etna\n(southeastern cone)\nlateral vent\nResort\nBiancavilla\nGiarre\nAcireale\nThe highly active Mount Etna in Italy sent red lava rolling down its flank on \nMarch 19, 2017. An astronaut onboard the ISS took the photograph below of the \nvolcano and its environs that night. City lights surround the mostly dark volcanic 51Volcanoes\nmountain. On the previous day, the Operational Land Imager (OLI) on Landsat 8 \nacquired this natural-color image (below) of an ash plume emanating from the summit and the flank.\nMarch 18, 2017\nplumeMt. Etna\n50 km\n52 Earth at NightMoonlight and MoonglintMoonlight and Moonglint\nMoon Phases\u2014Persian Gulf\nThe VIIRS DNB on the Suomi NPP satellite captured these nighttime views of the Per -\nsian Gulf region on September 30 and October 5, 10, and 15, 2012. Each image includes \nan inset of the Moon in one of four different phases. September 30 shows the Persian Gulf by the light of the full Moon; October 15 shows the effects of a new Moon. As the amount of moonlight decreases, some land surface features become harder to detect, but the lights from cities and ships become more obvious.", + " As the amount of moonlight decreases, some land surface features become harder to detect, but the lights from cities and ships become more obvious. Urbanization is most apparent along the northeastern coast of Saudi Arabia, in Qatar, and in the United Arab Emirates (UAE). In Qatar and UAE, major highways can even be discerned by nighttime lights. 53Moonlight and Moonglint\nSeptember 30, 2012\nDubai\nDoha\nRiyadh\n150 km\nOctober 5, 2012\nDubai\nDoha\nRiyadh\n150 km\nOctober 10, 2012\nDubai\nDoha\nRiyadh\n150 km\nOctober 15, 2012\nDubai\nDoha\nRiyadh\n150 km\n54 Earth at NightMoonlight and MoonglintMoonglint\u2014Elba and the Mediterranean\nA crew member onboard the ISS took this photograph of the northern Mediterranean \nSea and some coastal Italian towns and islands on October 17, 2013. The reflection of \nthe Moon on the sea surface\u2014moonglint\u2014reveals highly complex patterns. The strongest reflection is near the center of the Moon\u2019s disc, which brightens the water around the island of Elba. In these complex patterns, the dark areas of the sea surface can some -\ntimes make islands, such as Montecristo and Pianosa, harder to see.", + " In these complex patterns, the dark areas of the sea surface can some -\ntimes make islands, such as Montecristo and Pianosa, harder to see.\nThe reflection off sea surfaces captures many different natural processes but also \nsome made by humans. North of Elba, waves trailing behind ships make a classic V-shaped pattern. The meandering line coming off Montecristo Island is an island wake, a result of alternating vortices of wind that develop on the downwind side of the island. This wake is the strongest evidence that a northeast wind was blowing (right to left in this image) on the night of the photo. A shorter, meandering wind pattern is being shed off Punta Ala on the mainland. Smoother surfaces, protected from wind, are usually brighter because they are a better mirror for moonlight.\nThe sea surface also displays numerous tight swirls known as gyres. The broad swath \nof parallel lines (top left) is probably part of the larger circulation of the sea, which usually experiences north-flowing currents around Elba. 55Moonlight and Moonglint\nOctober 17, 2013\nElba\nMontecristo\nPianosa\nship wake\nPunta Ala\nPiombino\n56 Earth at ", + " 55Moonlight and Moonglint\nOctober 17, 2013\nElba\nMontecristo\nPianosa\nship wake\nPunta Ala\nPiombino\n56 Earth at NightClouds and LightningClouds and Lightning\nMarine Layer Clouds\u2014California\nOn September 27, 2012, the VIIRS DNB on the Suomi NPP satellite captured this \nnighttime view of low-lying marine layer clouds along the coast of California (see below). \nAn irregularly shaped patch of high clouds hovered off the coast of California, and moonlight caused the high clouds to cast distinct shadows on the marine layer clouds below. VIIRS acquired the image when the Moon was in its waxing gibbous phase.\nSeptember 27, 2012\nNevada\nCalifornia\nLos Angeles\nSan Francisco\nLas Vegas\nPacific Ocean\nshadow from moonlight\n100 km\nmarine layer clouds 57Clouds and Lightning\nLow clouds can pose serious hazards for air and ship traffic, and satellites have \nhad difficulty detecting them in the past. To illustrate this, the image on this page shows \nthe same scene as viewed by the VIIRS thermal infrared band used by meteorologists to monitor clouds at night.", + " To illustrate this, the image on this page shows \nthe same scene as viewed by the VIIRS thermal infrared band used by meteorologists to monitor clouds at night. Only high clouds are visible; the low clouds do not show up at all because they are roughly the same temperature as the ground.\nSeptember 27, 2012\nNevada\nCalifornia\nhigh clouds\nPacific Ocean\n100 km\n58 Earth at NightClouds and Lightning\nJanuary 23, 2016\nWashington, D.C.\nNew York\nBoston\n100 km\n 59Clouds and Lightning\nA Blizzard by Moonlight\u2014Northeastern United States\nA massive winter storm system pummeled the eastern United States in late January 2016, \nwhen two low-pressure systems merged into a potent nor\u2019easter that dropped heavy \nsnow from Virginia to New England. By late afternoon on January 23, snowfall totals were approaching records in several states and hurricane-force winds were battering the coastlines and leading to serious flooding. The VIIRS DNB on the Suomi NPP satellite acquired this image of the storm system at 2:15 a.m. Eastern Standard Time on January 23, 2016. Notice that the lights from the major cities along the East Coast of the United States are blurred in ", + " Notice that the lights from the major cities along the East Coast of the United States are blurred in places by cloud cover.60 Earth at NightClouds and LightningSensing Lightning from the ISS\u2014Saudi Arabia and Bolivia\nDecember 12, 2013\nKuwait City\nKhafji\n(Saudi Arabia)\nJafar Al Batom\n(Saudi Arabia)\nlightning\nLightning flashes about 50 times per second across all of Earth\u2019s atmosphere. That\u2019s \n4.3 million times a day and roughly 1.5 billion times a year. The December 12, 2013 pho -\ntograph (below, left), snapped by an astronaut onboard the ISS, shows a white flash of 61Clouds and Lightning\nlightning amidst the yellow city lights of Kuwait and Saudi Arabia. On another occasion, \nseen in the January 9, 2011 photograph (below, right), an astronaut onboard the ISS \ncaptured a close-up of a lightning flash beneath a thunderhead over Bolivia.\nJanuary 9, 201162 Earth at NightClouds and LightningLightning\u2014Japan\nAstronauts captured this photograph of \nsouth-central Japan on August 16, 2014, as \nthe ISS was passing over the western Pacific Ocean.", + "9, 201162 Earth at NightClouds and LightningLightning\u2014Japan\nAstronauts captured this photograph of \nsouth-central Japan on August 16, 2014, as \nthe ISS was passing over the western Pacific Ocean. The brightest spot in the center of the photo is the port city of Nagoya, the third largest metropolitan area in Japan. To the left, the lights of the urban areas of Osaka, Kyoto, and Wakayama pierce through clearings in the cloud cover. (Note that north is to the upper right in this image.) Much of the scene is obscured by cloud cover, although a lightning storm lights up the clouds to the north. Note how the light -\nning is much whiter than the yellow and orange hues of city lights.\nAugust 16, 2014\nWakayama\nKyoto\nOsaka\n 63Clouds and Lightning\nlightning\nNagoya64 Earth at NightClouds and LightningElusive Sprite Captured from the ISS\u2014Southeast Asia\nThey were once so elusive that scientists gave them a mystical name. Red sprites \nare short-lived, red flashes that occur about 50 miles (80 kilometers) up in the atmosphere. \nWith long, vertical tendrils like a jellyfish, these electrical discharges can extend 12 to 19 miles (20 to 30 kilometers) into the atmosphere and are connected to ", + " \nWith long, vertical tendrils like a jellyfish, these electrical discharges can extend 12 to 19 miles (20 to 30 kilometers) into the atmosphere and are connected to thunderstorms and lightning.\nThese three serial images, part of a time-lapse movie collected from 13:41 to 13:47 UTC \non April 30, 2012, show a red sprite (center image) captured with a digital camera by \nApril 30, 2012\nlightning\nsprite\n 65Expedition 31 astronauts on the ISS as they traveled southeast from central Myanmar \n(Burma) to just north of Malaysia. Red sprites are difficult to observe because they last for just a few milliseconds and occur above thunderstorms\u2014meaning they are usually blocked from view on the ground by the very clouds that produce them. The sprites send pulses of electrical energy up toward the edge of space\u2014into the electrically charged layer known as the ionosphere\u2014instead of down to Earth\u2019s surface. They are rich with radio noise, and can sometimes occur in bunches.\nClouds and Lightning66 Earth at NightClouds and Lightning\nJanuary 12, 2015\neye\nairglow\nsolar panel\neye\n 67Clouds and Lightning\nThe Electric Eye of Cyclone Bansi\u2014Indian Ocean\nThough these images ", + "\nClouds and Lightning66 Earth at NightClouds and Lightning\nJanuary 12, 2015\neye\nairglow\nsolar panel\neye\n 67Clouds and Lightning\nThe Electric Eye of Cyclone Bansi\u2014Indian Ocean\nThough these images may look like they come from a science fiction movie, they are \nin fact photographs of tropical cyclone Bansi as seen at night by astronauts on the ISS on \nJanuary 12, 2015. The images were taken when the ISS was east of Madagascar. The dim swirl of the cloud bands covers the ocean surface in both night images. The eye of the cyclone is brilliantly lit by lightning in or near the eye wall. The low-light settings of the camera used to take the image accentuate the contrast.\nIn the bottom image, the camera also accentuates the yellow-green airglow above Earth\u2019s \nlimb, an atmospheric phenomenon frequently reported by astronauts. Stars appear above the airglow layer, and the solar panels of a docked Russian spacecraft jut into the upper left of the image.68 Earth at NightSnow and IceSnow and Ice\nSnow Cover\u2014Great Lakes Region, \nUnited States\nAn Arctic air mass brought snow to \ncommunities around the Great Lakes on \nDecember 14, 2016.", + "68 Earth at NightSnow and IceSnow and Ice\nSnow Cover\u2014Great Lakes Region, \nUnited States\nAn Arctic air mass brought snow to \ncommunities around the Great Lakes on \nDecember 14, 2016. The lake-effect snow came on the heels of an earlier accumulation that dumped several feet of snow in some areas the week before. The VIIRS DNB on the Suomi NPP satellite captured this image on December 14. The instrument can detect faint light sources\u2014in this case, the reflection of the full Moon on the fresh snow. Clouds blur the landscape in the bottom left part of the image. Parallel rows of clouds, called cloud streets, appear above Lake Huron and Lake Superior. These clouds are created by cold, dry air blowing over a lake and accumulating water vapor.\nMinnesota\nIowa\nDecember 14, 2016\nclouds\nMinneapolis\n100 km\n 69Snow and Ice\nWisconsin\nIowa\nMichigan\nIndiana\n Illinois\nOhio\nLake\nErieLake\nMichgan\nLake\nHuron\nLake\nSuperior\nsnow\nclouds\nclouds\ncloud \nstreets\ncloud \nstreets\nMinneapolis\nChicago\nMilwaukee\nDetroit70 Earth at NightSnow and Ice\nOctober 30, 2012\nsea ice\nopen water\nclouds\n100 km\n 71Polar Darkness\u2014The Arctic\nScientists watched the Arctic with particular \ninterest in the summer ", + "snow\nclouds\nclouds\ncloud \nstreets\ncloud \nstreets\nMinneapolis\nChicago\nMilwaukee\nDetroit70 Earth at NightSnow and Ice\nOctober 30, 2012\nsea ice\nopen water\nclouds\n100 km\n 71Polar Darkness\u2014The Arctic\nScientists watched the Arctic with particular \ninterest in the summer of 2012 when the areal \nextent of Arctic sea ice set a new record low. The behavior of sea ice following such a low extent also interests scientists, but as Arctic sea ice was advancing in the autumn of 2012, so was polar darkness. Fortunately, the VIIRS DNB on the Suomi NPP satellite can see in the dark and acquired this nighttime view of sea ice north of Russia and Alaska on October 30, 2012. During polar darkness (i.e., winter months), VIIRS DNB data allow scientists to observe sea ice formation and snow cover extent at the highest latitudes and to identify clear water where ships can safely navigate.\nAlaska\nNorth SlopeSnow and Ice72 Earth at NightAurorasAuroras\nAurora Borealis\u2014Midwestern United States\nWhen viewed from the ISS, the night skies are illuminated with light from many \nsources. For example, the Midwestern United States presents a nighttime appearance \nwhen viewed from orbit, not unlike a patchwork quilt.", + " For example, the Midwestern United States presents a nighttime appearance \nwhen viewed from orbit, not unlike a patchwork quilt. The artificial light from human settlements appears with a characteristic yellow tinge, and the green light of the \nSeptember 29, 2011\naurora\nOmaha\nDes Moines\nMinneapolis-St. Paul 73\nChicago\nSt. Louis\nlightning\nAppalachian Mts.\nAuroras\naurora borealis shines brightly\u2014even seeming to reflect off Earth\u2019s surface in Canada\u2014\nin this view from the ISS taken on September 29, 2011. A small white patch of light is almost certainly lightning from a storm on the East Coast (image right). Part of the ISS appears across the top of the image.74 Earth at NightAuroras\nLooking Down at the Aurora\u2014Northern Hemisphere\nEven Earth\u2019s skies were celebrating St. Patrick\u2019s Day on March 17, 2015, as a severe \ngeomagnetic storm\u2014the strongest of the past decade\u2014painted the sky with green, red, \nand blue auroras from New Zealand to Alaska. The storm conditions provided a fantastic opportunity for aurora viewing from above.", + "storm\u2014the strongest of the past decade\u2014painted the sky with green, red, \nand blue auroras from New Zealand to Alaska. The storm conditions provided a fantastic opportunity for aurora viewing from above. On Sunday, March 15, a coronal mass ejection released large amounts of plasma and an accompanying magnetic field from the Sun, causing the aurora borealis to reach as far south as the central and southern United States, as shown on the image at right. The VIIRS DNB on the Suomi NPP satellite acquired this view of the aurora around 1:30 a.m. Eastern Daylight Time on March 18, 2015. Auroras appear as white streaks over Hudson Bay, southern Canada, and the northern United States. 75Auroras\nMarch 18, 2015\naurora\nCanada\nHudson\nBay\nMontreal\nU.S.\nMinneapolis/St. Paul\n250 km\n76 Earth at NightAurorasAurora Lights Up the Night\u2014Antarctica\nOn July 15, 2012, the VIIRS DNB on the Suomi NPP satellite captured this night-\ntime view of the aurora australis over Antarctica\u2019s Queen Maud Land and the Princess \nRagnhild Coast. The slightly jagged appearance of the auroral lines is a function of the rapid dance of the energetic particles at the same time that the satellite is moving and \nJuly 15, 2012\n", + " The slightly jagged appearance of the auroral lines is a function of the rapid dance of the energetic particles at the same time that the satellite is moving and \nJuly 15, 2012\nAntarctica\naurora\nSouthern Ocean\n200 km 77Auroras\nthe VIIRS sensor is scanning. The white box in the left image depicts the area shown \nin the close-up image at right (below). Light from the aurora was bright enough to illu-minate the ice edge between the ice shelf and the Southern Ocean. At the time, Ant-arctica was locked in midwinter darkness and the Moon was a waning crescent that provided little light.\nsea ice\nice shelf\naurora\nPrincess Ragnhild Coast\n50 km78 Earth at NightAirglowAirglow\nAirglow\u2014Australia\nOn October 7, 2018, an astronaut onboard \nthe ISS shot this photograph at an altitude of \nabout 250 miles (400 kilometers) over Australia. In this view, stars appear more numerous along the image center where the plane of the disk-shaped Milky Way galaxy extends into space. The orange colors enveloping Earth are known as airglow\u2014diffuse bands of light that stretch roughly 50 to 400 miles (80 to 645 kilometers) into our atmosphere.", + " The orange colors enveloping Earth are known as airglow\u2014diffuse bands of light that stretch roughly 50 to 400 miles (80 to 645 kilometers) into our atmosphere. This type of airglow is known as chemiluminescence, or nightglow. More than just a pretty light show, airglow reveals some of the workings of the upper reaches of our atmosphere. Studying \nairglow can help scientists learn about the movement of particles near the interface between Earth and space, including the connections between space weather and Earth weather. Airglow reveals some of the conditions of the upper atmosphere, such as its temperature, shape, and the amounts of different types of gases.\nOctober 7, 2018 79Airglow\n80 Earth at NightAirglow\nApril 2, 2014\nMoscow\nHelsinki\nSt. Petersburg\n 81Airglow\nNight Colors\u2014Russia\nA thin green line of the aurora borealis crosses \nthe top of this photograph, taken by an astronaut \non the ISS on April 2, 2014. The Moon appears as a white disc just above the aurora. Airglow appears as a blue-white cusp along Earth\u2019 s limb. Russia\u2019s capital city, Moscow, makes a splash of yellow light and is easily recognized by the radial pattern of its ", + " Russia\u2019s capital city, Moscow, makes a splash of yellow light and is easily recognized by the radial pattern of its highways. Other cities are Nizhny Novgorod [250 miles (400 kilometers) from Moscow], Saint Petersburg [388 miles (625 kilometers) away], and Finland\u2019s capital city Helsinki [about 545 miles (876 kilometers) away].\nNizhny Novgorod\naurora\nairglow82 Earth at NightAirglowWaves in Airglow\u2014Texas\nApril 15, 2012\nTexas\nOklahoma\nNew \nMexico\nMexico\nDallas-Fort Worth\nHouston\nSan Antonio\nGulf of \nMexico\nwaves\n100 km\nAs shown in the left image (below), in April 2012 waves in Earth\u2019s airglow spread \nacross the nighttime skies of Texas like ripples in a pond. In this case, the waves were \nprovoked by a massive thunderstorm. The VIIRS DNB on the Suomi NPP satellite captured these glowing ripples in the night sky on April 15, 2012. The right image (next page) 83Airglow\nshows the thunderstorm as observed by a VIIRS thermal infrared band. This thermal \nband, which is sensitive only to heat emission, is not sensitive to the subtle visible-light wave structures seen by the VIIRS DNB.", + " This thermal \nband, which is sensitive only to heat emission, is not sensitive to the subtle visible-light wave structures seen by the VIIRS DNB. Clouds appear white because they are cooler than Earth\u2019s surface.\nApril 15, 2012\nthunderstorm\nTexas\nNew \nMexico\n Oklahoma\nMexico\nGulf of \nMexico100 km\n84 Earth at NightHuman Light Sources\nViewing Earth at night from space reveals a distinctly human story. As one gazes \nupon the points of light in images captured from orbiting sensors, discerning \neyes begin to see the fingerprints of human presence: from the glow of megacities and the pinpoints of bright light produced by fleets of fishing boats at sea, to gas flares glowing like candles in the darkness, lights show where we have made our homes, established industries, mined natural resources, and built roads. Patterns are everywhere.\nIt quickly becomes evident looking at nightlight data that human activity is highly \ninfluenced by Earth\u2019s natural features and processes, or physical geography. For \nexample, the vital link between humans and water is quite clear.", + "\nIt quickly becomes evident looking at nightlight data that human activity is highly \ninfluenced by Earth\u2019s natural features and processes, or physical geography. For \nexample, the vital link between humans and water is quite clear. Nearly half the world\u2019s population lives along or near coastlines. Others live along rivers and freshwater bodies. Most population centers tend to be along coastlines, near rivers, or near major over -\nland transportation hubs (e.g., highways, railroads) that cut slivers of light through the darkness between cities. In general, our species tends to avoid settling in regions with harsh climates (or if we do, we set about transforming them into something more pleasant), in mountainous areas with steep terrain, and where basic resources like food and water are limited (e.g., deserts). Accordingly, more light is clustered around regions where the climate is moderate, terrain is flat, resources are plentiful, and soil is fertile.\nAreas with the brightest light are generally the most urbanized\u2014but not necessarily \nthe most populated. When a city or country is thriving, electricity is used to keep \nbusinesses, schools, and factories bustling with activity.", + " When a city or country is thriving, electricity is used to keep \nbusinesses, schools, and factories bustling with activity. Lights show how far urban sprawl extends, as well as areas where growth is occurring or where growth has not yet occurred. Cities usually have many people concentrated in a small area, so electricity \nusage is high. Poor areas may have large populations but low availability or use of elec -\ntric lights. These signals from anthropogenic nightlights provide clear indicators of how we\u2019ve settled our planet and\u2014with some analysis\u2014what we do with our energy.[Opposite page] This \nphotograph, taken by an \nastronaut from the ISS on \nDecember 10, 2017, shows the city of Bangkok, the capital and largest city in Thailand, illuminated by city lights. The adjacent waters of the Andaman Sea and Gulf of Thailand are illuminated by hundreds of green lights on fishing boats. Fishermen use the lights to attract plankton and fish, the preferred diet of com -\nmercially important squid. As the bait swims to the surface, the squid follow to feed and get caught by fishermen.", + " Fishermen use the lights to attract plankton and fish, the preferred diet of com -\nmercially important squid. As the bait swims to the surface, the squid follow to feed and get caught by fishermen. The same fishing \npractices are used off the Atlantic coast of South America. The border between \nThailand and neighboring Cambodia to the east is distinguished by a marked difference in the number of city lights. Cambodia is less urbanized and its popula-tion is smaller than that of neighboring countries. The majority of Cambodia\u2019s population lives in rural farming areas where elec-tricity is sparse. 85\nDecember 10, 2017\nBangkok\nAndaman Sea\nGulf of Thailand\nISS Robotic Arm\nPhnom\nPenh\nfishing boats\n86 Earth at NightUrban StructureUrban Structure\nConstrained by Geography\u2014Reno, Nevada\nStyling itself as \u201cthe biggest little city in the world,\u201d the city of Reno, Nevada, \nis located in Truckee Meadows, along the expansion-limiting eastern foothills of the \nSierra Nevada near the California border. It forms part of the Reno\u2013Sparks metro-politan area\u2014the largest such area in northern Nevada, and the second largest in the state after Las Vegas.", + " It forms part of the Reno\u2013Sparks metro-politan area\u2014the largest such area in northern Nevada, and the second largest in the state after Las Vegas.\nOn January 28, 2013, when an astronaut took this photograph, the Moon was in a \nwaning gibbous phase (98 percent of full), providing enough illumination of the ground so that the topography surrounding the Reno\u2013Sparks area (accentuated by snow cover) is clearly visible.\nThe relatively isolated nature of the city within the surrounding terrain is highlighted \nin this nighttime image taken from the ISS. The major industrial and commercial areas of both Reno and Sparks are brightly lit at image center. The major street grid is visible as orange linear features adjacent to these industrial/commercial areas; in contrast, the more outlying residential areas appear dark. The Reno\u2013Tahoe International Airport is visible as a dark, dagger-shaped region. 87Urban Structure\nJanuary 28, 2013\n\u2014Sparks\nresidential\nresidential\nairport\nReno\nsnow\nsnow88 Earth at NightUrban StructureGrid of City Blocks\u2014Phoenix, Arizona\nLike many large urban areas of the central and western United States, the \nPhoenix ", + " 87Urban Structure\nJanuary 28, 2013\n\u2014Sparks\nresidential\nresidential\nairport\nReno\nsnow\nsnow88 Earth at NightUrban StructureGrid of City Blocks\u2014Phoenix, Arizona\nLike many large urban areas of the central and western United States, the \nPhoenix metropolitan area is laid out along a regular grid of city blocks and streets. \nWhile visible during the day, this grid is most evident at night, when the pattern of street lighting is clearly visible from the low-Earth-orbit vantage point of the ISS.\nThis astronaut photograph, taken on March 16, 2013, includes parts of several \ncities in the metropolitan area, including Phoenix (image right), Glendale (center), \nand Peoria (left). While the major street grid is oriented north-south, the northwest- southeast oriented Grand Avenue cuts across the three cities at image center. Grand Avenue is a major transportation corridor through the western metropolitan area; the lighting patterns of large industrial and commercial properties are visible along its length. Other brightly lit properties include large shopping centers, strip malls, and gas stations, which tend to be located at the intersections of north-south and east-west trending streets.", + " Other brightly lit properties include large shopping centers, strip malls, and gas stations, which tend to be located at the intersections of north-south and east-west trending streets.\nThe urban grid encourages growth outwards along a city\u2019s borders by providing \noptimal access to new real estate. Fueled by the adoption of widespread personal automobile use during the twentieth century, the Phoenix metropolitan area today includes 25 other municipalities (many of them largely suburban and residential) linked by a network of surface streets and freeways.\nWhile much of the land area highlighted in this image is urbanized, there are \nseveral noticeably dark areas. The Phoenix Mountains are largely public parks and recreational land. To the west, agricultural fields provide a sharp contrast to the lit streets of residential developments. The Salt River channel appears as a dark ribbon within the urban grid. 89\nUrban Structure\nagricultural\nfields\nPhoenix\nMountains\nSalt River\nChannel\nGrand Avenue\nPhoenix\nGlendale\nPeoria\nMarch 16, 201390 Earth at NightUrban StructureUnpopulated Slopes of an \nActive Volcano\u2014Naples, Italy\nAn astronaut onboard the ISS took this \nphotograph ", + "Channel\nGrand Avenue\nPhoenix\nGlendale\nPeoria\nMarch 16, 201390 Earth at NightUrban StructureUnpopulated Slopes of an \nActive Volcano\u2014Naples, Italy\nAn astronaut onboard the ISS took this \nphotograph of the city lights of Naples and \nthe Campania region of southern Italy on January \n30, 2017. The Naples region is one of the brightest in Italy. Roughly three million people live in and around this metropolitan area.\nThe large black circular area in the photo is Mount Vesuvius, the only active \nvolcano on Europe\u2019s mainland. Although any volcanic activity can endanger surrounding communities, eruptive pyroclastic flows of superheated ash and gas are among the most dangerous, moving at speeds of hundreds of kilo-meters per hour. Vesuvius has erupted on numerous occasions throughout history. Probably the most famous of those eruptions occurred in 79 A.D., when pyroclastic flows destroyed the cities of Pompeii and Herculaneum, trapping more than 16,000 people. Such historic catastrophes\u2014and the fact that 600,000 people currently live in the immediate vicinity\u2014are why the volcano is one of the most heavily monitored in the world, with several dozen sensors located at many points on and ", + "the fact that 600,000 people currently live in the immediate vicinity\u2014are why the volcano is one of the most heavily monitored in the world, with several dozen sensors located at many points on and around the cone.\nThe different colors of lights in the scene reflect some of the history of \ndevelopment in the area. The green lights are mercury vapor bulbs, an older variety that has been replaced in newer developments by yellow-orange sodium bulbs. To the northeast, the lightless gaps between the homes and businesses are agricultural fields. The bright yellow-orange complex amidst the fields is the Consorzio Intercomunale dei Servizi, the largest commercial \nfacility in Europe. A regional view of Italy appears on \npage 92.\nAgnano\nBay of\nNaples\nNaples\nJanuary 30, 2017 91\nUrban Structure\nMt. Vesuvius\nairport\nfields\nfields\nConsorzio \nIntercomunale \ndei Servizi\nHerculaneum\nPompeii\n92 Earth at NightUrban StructureDazzling Coastlines\u2014Italy\nCity lights clearly delineate the boot of Italy in this panorama taken by astronauts \nonboard the ISS on October 21, 2014. Looking east on this clear night, the pattern \nof nightlights shows populations concentrated mainly along the coastlines, but ", + " Looking east on this clear night, the pattern \nof nightlights shows populations concentrated mainly along the coastlines, but also in the Po River Valley of northern Italy. Some of the brightest clusters of lights are Rome and nearby Naples (see image on previous page), along with the island cities of \nOctober 21, 2014\nairglow\nCagliari\nRome\nPo Valley\nNaples\nSardinia 93\nUrban Structure\nCagliari on Sardinia and Catania on Sicily. The small, dark, circular patch dangerously \nclose to Catania marks the unpopulated slopes of the active volcano Etna. Hazy lights, as over the Po Valley and Rome, probably indicate thin clouds above. The island of Malta, which is the tenth smallest and fifth most densely populated country in the world, appears in the lower right. Airglow is vivid in this night shot.\nMediterranean Sea\nMalta\nISS solar panels\nCatania\nMt. Etna\nNaples\nSicily\n94 Earth at NightUrban StructureLiving on Fertile Land\u2014Nile River, Egypt\nThe Nile River was the center of ancient \nEgyptian civilization\u2014the lifeline for one of his-\ntory\u2019s first great empires.", + " Etna\nNaples\nSicily\n94 Earth at NightUrban StructureLiving on Fertile Land\u2014Nile River, Egypt\nThe Nile River was the center of ancient \nEgyptian civilization\u2014the lifeline for one of his-\ntory\u2019s first great empires. Annual floods helped bring silt to the soil, which allowed generation after generation to grow crops. Today\u2019s nighttime \nlights show clearly that Egypt\u2019s population is, as it was in ancient history, almost completely concentrated along the Nile Valley\u2014a small percentage of the country\u2019s land area, the remainder of which is desert.\nThe Nile River and its heavily populated delta look like a brilliant, long-\nstemmed flower in this photograph of the southeastern Mediterranean Sea, as seen from the ISS on October 28, 2010. The Cairo metropolitan area forms a particularly bright base for the flower. The smaller cities and towns within the Nile Delta are harder to see during the day amid the dense agri-cultural vegetation. However, these settled areas and the connecting roads between them become clearly visible at night. Similarly, urbanized regions and infrastructure along the Nile River become apparent.", + " However, these settled areas and the connecting roads between them become clearly visible at night. Similarly, urbanized regions and infrastructure along the Nile River become apparent.\nAnother brightly lit region visible along the eastern coastline of the \nMediterranean is the Tel Aviv metropolitan area in Israel (image right). To the east of Tel Aviv lies Amman, Jordan. The two major water bodies that define the western and eastern coastlines of the Sinai Peninsula\u2014the Gulf of Suez and the Gulf of Aqaba\u2014are outlined by lights along their coastlines (image lower right). The city lights of Paphos, Limassol, Larnaca, and Nicosia are visible on the island of Cyprus (image top).\nScattered blue-grey clouds cover the Mediterranean Sea and the Sinai, \nwhile much of northeastern Africa is cloud-free. A thin yellow-brown band tracing Earth\u2019s curvature at image top is airglow.\nOctober 28, 2010\nISS\nsolar panel\n 95\nUrban Structure\nEgypt\nairglow\nMediterranean Sea\nCairo\nGulf of Suez\n Gulf of Aqaba\nTel Aviv\nAmman\nCyprus\nPaphos\nLimassol\nNicosia\nLarnaca96 Earth at NightUrban StructureThe Winding Seine River and the City of Light\u2014Paris, France\nAround local midnight on April 8, 2015,", + "95\nUrban Structure\nEgypt\nairglow\nMediterranean Sea\nCairo\nGulf of Suez\n Gulf of Aqaba\nTel Aviv\nAmman\nCyprus\nPaphos\nLimassol\nNicosia\nLarnaca96 Earth at NightUrban StructureThe Winding Seine River and the City of Light\u2014Paris, France\nAround local midnight on April 8, 2015, astronauts onboard the ISS took this \nphotograph of Paris, often referred to as the \u201cCity of Light.\u201d When viewed from \nlow Earth orbit, the pattern of the street grid dominates the Parisian night, providing a \ncompletely different set of visual features from those visible during the day. For instance, the winding Seine River is a main visual cue by day, but here the thin black line of the river is hard to detect until you focus on the strong meanders and the streetlights on both banks.\nThe brightest boulevard in the dense network of streets is the Avenue des Champs-\n\u00c9lys\u00e9es, the historical axis of the city, as designed in the seventeenth century. This grand avenue joins the site of the former royal Palais des Tuileries\u2014whose gardens appear as a dark rectangle on the river\u2014to the star-like meeting place of eleven major boulevards at the Arc de Triomphe. This famous plaza is also referred to as the \u00c9toile, or \u201cstar.", + "whose gardens appear as a dark rectangle on the river\u2014to the star-like meeting place of eleven major boulevards at the Arc de Triomphe. This famous plaza is also referred to as the \u00c9toile, or \u201cstar.\u201d\nThe many forested parks of Paris stand out as black polygons\u2014such as the \nBois de Boulogne and Bois de Vincennes. The Bois de Boulogne is crisscrossed by a few roads and lighted paths.\nOrly airport is distinguished by very bright lights next to the dark areas or runways \nand surrounding open land. Paris\u2019s great ring road, the Boulevard P\u00e9riph\u00e9rique, encloses the city center. 97Urban Structure\nApril 8, 2015\nArc de Triomphe\nBois de\nBoulogne\nSeine River\nOrly Airport\nPalais des Tuileries\nBois de\nVincennes\nCharles de Gaulle Airport\nBoulevard P\u00e9riph\u00e9rique\nAvenue des Champs-\u00c9lys\u00e9es\n98 Earth at NightUrban StructureLighting Paths to Oil\u2014Qatar\nIn the photograph below, taken from the ISS on October 13, 2012, the nightlights \nof Qatar show informative demographic detail that is very difficult to discern in daylight \nimages\u2014especially in deserts, where even large cities can be hard to see. The brightest group of lights at image center is the capital city, Doha, with the neighboring smaller ports ", + " The brightest group of lights at image center is the capital city, Doha, with the neighboring smaller ports of Ad-Dahirah and Umm Sa\u2019id to the north and south. (Note that north is to the left \nin this image, due to the path of the ISS orbit.) Highways are clearly visible leading west from the capital to the Dukhan oil fields, to Saudi Arabia, and to the north of the country where\u2014judging by the lack of nightlights\u2014the population is probably very low. The \nrelatively minor coastal road between the oil fields and the Saudi frontier also stands out.\nOctober 13, 2012\nPersian Gulf\nGulf of Bahrain\nDoha\nQatar\nSaudi Arabia\nUmm Sa\u2019id\nAd-Dahirah\nManama\nBahrain\ncamel race track\nDukhan oil field\n 99Urban Structure\nAlmost the entire island nation of Bahrain appears at lower left, with the capital city \nof Manama nearly as bright as the lights of Doha. The difference in light intensity reflects \na difference in population: Doha has 1.45 million inhabitants, while the dense Manama metropolitan area has a population of 1.", + " The difference in light intensity reflects \na difference in population: Doha has 1.45 million inhabitants, while the dense Manama metropolitan area has a population of 1.2 million. The thumb-shaped Qatari penin-sula, so well-known in Middle Eastern geography, does not show up at all in this nighttime photograph. However, in the image below, the low-light imaging bands of VIIRS on the Suomi NPP satellite showed the Qatari peninsula and the long arm of the Gulf of Bahrain on a moonlit night. The image was acquired during the early morning hours of September 30, 2012, two weeks before the ISS photograph at left.\nSeptember 30, 2012\nSaudi Arabia\nRiyadh\nQatar\nOman\nUnited Arab\nEmirates\n ISS photograph footprint\n100 km100 Earth at NightUrban StructureSnaking Along Canyon Cliffs\u2014Haifa, Israel\nNovember 29, 2015Mediterranean Sea\nHaifaPort of Haifa\ncanyon cliffsCape Carmel\nIn geography training, astronauts are taught to concentrate on the shapes of \ncoastlines because they are a first-order visual cue when circling the planet\u2014and often \nuniquely shaped. The nose of Cape Carmel and the bay that protects the Port of Haifa are shapes that can tell crews where they ", + " The nose of Cape Carmel and the bay that protects the Port of Haifa are shapes that can tell crews where they are. In the daylight image of the port city of Haifa on Israel\u2019s Mediterranean coast (below), the strong visual line of the coast \ncontrasts with the subtle city colors. The night image (right) shows different city 101Urban Structure\nneighborhoods in a way that is difficult to see during the day. The brilliant port lights \ncontrast with somewhat dimmer residential areas. Straight roads of the older residential neighborhoods are easily distinguished from the winding roads that follow the canyon cliffs. The industrial district just east of the port has areas of green and blue lights and a less-dense street pattern. Surrounding farmlands are so dark that they can be confused with the sea.\nCape Carmel\nMediterranean Sea\nHaifaPort of HaifaDecember 14, 2015\ncanyon cliffs102 Earth at NightUrban StructureDifferences in Socio-Economic Strategies\u2014Korean Peninsula\nUnlike daylight images, lights at night illustrate dramatically the economic signifi -\ncance of cities, as gauged by relative size.", + "Sea\nHaifaPort of HaifaDecember 14, 2015\ncanyon cliffs102 Earth at NightUrban StructureDifferences in Socio-Economic Strategies\u2014Korean Peninsula\nUnlike daylight images, lights at night illustrate dramatically the economic signifi -\ncance of cities, as gauged by relative size. Usually, nightlights can be used to gauge \na region\u2019s population: more people mean more lights. But the Korean Peninsula has a different story to tell. Flying over East Asia, astronauts on the ISS took this night image of the Korean Peninsula on January 30, 2014. In this north-looking view, it is immediately obvious that Seoul is a major city and that the port of Gunsan is minor by comparison. There are 25.6-million people in the Seoul metropolitan area\u2014more than half of South Korea\u2019s citizens\u2014while Gunsan\u2019s population is 280,000. North Korea is \nJanuary 30, 2014\nYellow SeaBo Hai Gulf\nChina\nNorth Korea\nPyongyang\nSeoul\n 103Urban Structure\nalmost completely dark compared to neighboring South Korea and China. Its capital \ncity, Pyongyang, appears like a small island, despite a population of 3.26 million (as of 2008). The light emission from Pyongyang is equivalent to the smaller towns in South Korea.", + "26 million (as of 2008). The light emission from Pyongyang is equivalent to the smaller towns in South Korea. Coastlines are often very apparent in night imagery, as shown by South Korea\u2019s eastern shoreline, but the coast of North Korea is difficult to detect. These differences are illustrated in the significant differences in per capita power consumption between the two countries, with South Korea at 10,162 kilowatt hours and North Korea at 739.\nSea of Japan\nNorth Korea\nSouth Korea\nGunsan\nSeoul\n104 Earth at NightUrban StructureNow You See Them, Now You Don\u2019t\u2014Argentina\n2016\nBuenos AiresSanta FeC\u00f3rdoba\nVilla Maria\nRio CuartoRosario\nSan LuisPan-American \nHighway\n50 kmWhen viewed from a satellite during the day (image below), the landscape of \ncentral Argentina shows barely a sign of human settlement. But by night (right image), \nthe evenly spaced points of light that reveal the presence of population centers \npuncture the darkness. It\u2019s no mistake that these lights appear roughly every 20 to 30 miles (every 30 to 50 kilometers), as many of the towns grew up around railway \nstations.", + " It\u2019s no mistake that these lights appear roughly every 20 to 30 miles (every 30 to 50 kilometers), as many of the towns grew up around railway \nstations. Maps from the middle of the twentieth century show the network of railway lines that gave shape to this curious configuration. Today, cars have outpaced trains in popularity, but the mark of the railways remains emblazoned in the terrain. Some tracks are still visible from space, such as those running parallel to the road between C\u00f3rdoba 105Urban Structure\nand Villa Maria. The same is true of the line that runs northeast through Rio Cuarto. \nHighways now connect the \u201cdots\u201d\u2014and these cities.\nThe nighttime image below shows Argentina\u2019s nightlights as observed by the VIIRS DNB \non the Suomi NPP satellite in 2016. The natural-color image on the left shows the same area by day with a cloudless composite from the MODIS instruments on the Aqua and Terra satellites. Fields for growing crops and raising cattle dominate the landscape. Most people reside in the country\u2019s main cities, including C\u00f3rdoba and Santa Fe.", + " Fields for growing crops and raising cattle dominate the landscape. Most people reside in the country\u2019s main cities, including C\u00f3rdoba and Santa Fe. A mere 10 percent of Argentina\u2019s population lives in rural areas, according to the Food and Agriculture Organiza -\ntion of the United Nations.\n2016\nBuenos AiresSanta FeC\u00f3rdoba\nVilla Maria\nRio CuartoRosario\nSan LuisPan-American \nHighway\n50 km106 Earth at NightUrban Structure A Well-Lit Border, Indus River\u2014Pakistan\nLike the Nile River Valley (shown on page 94), the Indus River \nBasin in Pakistan is an ancient seat of civilization. In spite of the \narid conditions that make it dif\nficult to grow food in the rest of \nthe region, people have lived and farmed in the river\u2019s fertile floodplain for millennia. In modern times, the sinuous shape of the Indus River emerges even in nighttime satellite imagery, based on data collected in 2016 by the VIIRS DNB on the Suomi NPP satellite. Though the river itself is only barely visible, the dark crops and vegetation growing along its banks help reveal the general shape of the river.", + " Though the river itself is only barely visible, the dark crops and vegetation growing along its banks help reveal the general shape of the river. The brighter, more reflective land beyond the dark band of farmed land is desert. Many of the largest cities and towns in Pakistan are clustered along the Indus. Karachi lies along the southernmost stretch of the river, near where it empties into the Arabian Sea at the Indus River Delta. Other brightly lit cities along the river include Hyderabad, Larkana, and Sukkur. The border between Pakistan and India stands out among the nightlights of this region. For security purposes, India has installed thousands of kilometers of floodlights along the border, a feature bright enough to be seen from the ISS.\n2016\nPakistan\nKarachi\n|\n100 km 107Urban Structure\nIndiaLarkana\n| \u2014 Sukkur\n\u2014 International Border\n\u2014 HyderabadIndus River \u2014108 Earth at NightUrban DevelopmentUrban Development\nLighting Paths\u2014Across the United States\nThe United States has more miles of roads than any other nation in the world\u2014 \n4.", + "Karachi\n|\n100 km 107Urban Structure\nIndiaLarkana\n| \u2014 Sukkur\n\u2014 International Border\n\u2014 HyderabadIndus River \u2014108 Earth at NightUrban DevelopmentUrban Development\nLighting Paths\u2014Across the United States\nThe United States has more miles of roads than any other nation in the world\u2014 \n4.1 million miles (6.6 million kilometers) to be precise, which is roughly 40 percent more \nthan second-ranked India. About 47,000 miles (75,639 kilometers) of those roads are part of the Interstate Highway \nSystem, established by President Dwight Eisenhower in the 1950s. The country also has 127,000 miles (204,000 kilometers) of railroad tracks and about 25,000 miles (40,000 kilometers) of navigable rivers and canals (not includ -\ning the Great Lakes). The imprint of that transportation web becomes easy to see at night.\nThe VIIRS DNB on the Suomi NPP satellite acquired this nighttime view (top image, right) of the continental \nUnited States on October 1, 2013. The roadway map (bottom image, right) traces the path of the major interstate highways, railroads, and rivers of the United States. Comparing the two images, you quickly see how the cities and settlements align with the transportation corridors.", + " Comparing the two images, you quickly see how the cities and settlements align with the transportation corridors. In the early days of the republic, post roads and toll roads for horse-drawn carts and carriages were built to connect eastern cities like Boston, New York, Baltimore, and Philadelphia, though relatively few travelers made the long, unlit journeys. Railroads became the dominant trans-portation method for people and cargo in the middle of the nineteenth century, establishing longer links across the Nation and waypoints across the Midwest, the Great Plains, and the Rockies. Had nighttime satellite images existed in that era, they probably would show only dim pearls of light around major cities in the east and scattered across the country; the strands of steel tracks and cobbled roads that connected them would be invisible from space.\nEventually, cars and trucks became the dominant form of transportation in the United States. Drivers then \nneeded roads and lighting to keep them safe on those roads.", + "\nEventually, cars and trucks became the dominant form of transportation in the United States. Drivers then \nneeded roads and lighting to keep them safe on those roads. As the Nation grew in the twentieth century, the devel -\nopment of new cities and suburbs often conformed to the path of the interstate highways, adding light along the paths between the cities.\nOver the years, the length of navigable rivers has been a constant, as is their relative lack of light. Even today \nthe only light seems to be the occasional port cities along riverbanks and the light of ships themselves. 109Urban Development\n500 kmOctober 1, 2013\ninterstate\nrailroadriver\nN500 km110 Earth at NightUrban DevelopmentThe Brightest Spot on Earth\u2014Las Vegas, Nevada\nThe city of Las Vegas was established in 1905. Its grassy meadows and arte-\nsian springs attracted settlers traveling across the arid Desert Southwest in the early \n1800s. In the 1930s, gambling became legalized and construction of the Hoover Dam began, resulting in the city\u2019s first growth spurt. Since then, Las Vegas has not stopped growing.", + " In the 1930s, gambling became legalized and construction of the Hoover Dam began, resulting in the city\u2019s first growth spurt. Since then, Las Vegas has not stopped growing. Population has reached nearly two million over the past decade, making this one of the fastest growing metropolitan areas in the world. The series of false-color Landsat images (top image grid, right) show the rapid urbanization of Las Vegas between 1972 and 2018. The city streets and other impervious surfaces appear gray, while irrigated vegetation appears red. Over the years, the expansion of irrigated vegetation (e.g., lawns and golf courses) has stretched the city\u2019s desert bounds.\nWhile the city is famous for its casinos and resort hotels\u2014Las Vegas bills itself as \n\u201cthe entertainment capital of the world\u201d\u2014the wider metropolitan area includes several other incorporated cities and unincorporated areas (not part of a state-recognized municipality). The surrounding darkness of the desert in the astronaut photo from the ISS (bottom image, right), taken on November 30, 2010, presents a stark contrast to the brightly lit street grid of the developed area.", + " The surrounding darkness of the desert in the astronaut photo from the ISS (bottom image, right), taken on November 30, 2010, presents a stark contrast to the brightly lit street grid of the developed area. The famous Vegas Strip is reputed to be the brightest spot on Earth due to the concentration of lights on its hotels and \ncasinos. The tarmac of McCarran International Airport appears dark by comparison, while the airstrips of Nellis Air Force Base on the northeastern fringe are likewise dark. The dark mass of Frenchman Mountain borders the city to the east. 111Urban Development\n1972 Landsat 1 MSS\n5 km\n1978 Landsat 2 MSS\n5 km\n1984 Landsat 5 TM\n5 km\n1990 Landsat 5 TM\n5 km\n1996 Landsat 5 TM\n5 km\n2002 Landsat 5 TM\n5 km\n2008 Landsat 5 TM\n5 km\n2015 Landsat 8 OLI\n5 km\n2018 Landsat 8 OLI\n5 km\nNovember 30, 2010\nFrenchman Mountain\nLas Vegas Strip\nMcCarran Airport\n Nellis Air Force Base\n112 Earth at NightUrban DevelopmentRapid Urban Growth\u2014Shanghai, China\nThese nighttime photographs taken by astronauts onboard the ISS reveal the \nunprecedented growth of Shanghai, China, between March 10, 2003 (below) and \nFebruary 28, 2018 (right).", + "Growth\u2014Shanghai, China\nThese nighttime photographs taken by astronauts onboard the ISS reveal the \nunprecedented growth of Shanghai, China, between March 10, 2003 (below) and \nFebruary 28, 2018 (right). The city of Shanghai sits along the delta banks of the Yangtze \nRiver along the eastern coast of China. The city proper is the world\u2019s most populous city.\nMarch 10, 2003\nShanghai\nPudongSuzhou\nYangtze RiverThe surge in China\u2019\ns urbanization began in the 1980s when the Chinese government \nbegan opening the country to foreign trade and investment. As markets developed in \n\u201cspecial economic zones,\u201d villages morphed into booming cities and cities grew into \nsprawling megalopolises. In 1982, Shanghai was a relatively compact industrial city of 12-million people; however, that number grew to 24 million by 2016. Much of the growth \nHuangpu River\n 113Urban Development\nhas occurred in new satellite developments like areas to the west of the city (for example, \nSuzhou, at the upper left of the image on the right).\nThe rapid pace of development has changed Shanghai\u2019s natural ecosystems. Wetlands \nin the region have declined due to sea level rise, erosion, dredging, and the construction of ", + " Wetlands \nin the region have declined due to sea level rise, erosion, dredging, and the construction of water-storage infrastructure. The creation of new coastal land\u2014by piling sediment onto tidal flats in a process called land reclamation\u2014has also played a key role. Despite the dizzying pace of urbanization, there are signs that Shanghai\u2019s growth may be tapering off. Officials announced that they will cap the city\u2019s population at 25 million in the hopes of easing the pressure on the environment, infrastructure, and city services.\nFebruary 28, 2018\nShanghai\nPudongYangtze River\nHuangpu River\nSuzhou\n114 Earth at NightUrban DevelopmentTurning Up the Lights\u2014India\nImages of Earth at night are being used by scientists and decision-makers to monitor \ngradual changes driven by urbanization and electrification. Upon assuming leadership \nin 2014, India\u2019s Prime Minister Narendra Modi pledged to electrify the nation and, in 2015, launched a program to electrify every village in the country. The image on the left (below) and the middle image (below) were created using two separate nightlight ", + " The image on the left (below) and the middle image (below) were created using two separate nightlight datasets from the VIIRS DNB on the Suomi NPP satellite for 2012 and 2016, respectively. The image on the right (below) shows the change in lighting intensity that results from \n2012\nLucknow, India\n2016\nLucknow, India\n 115Urban Development\ncombining the 2012 and 2016 images. Purple shades indicate areas that have more \nlights in 2016 than in 2012. Huge swaths of northern India\u2014relatively dark in the 2012 night image\u2014are lit up in the image from 2016. Orange shades indicate areas that have become dimmer since 2012. The dark area to the north (top) of the images is Nepal, which is almost completely lacking in observable lighting, clearly demonstrating the dif -\nferences in stages of development between the two countries. To view global changes \nin lighting intensity from 2012 to 2016, see \npage 26 of this volume.\nLucknow, India\n2012 - 2016 Change\nLucknow, India\nlights out no change new lights\nChange in Nightlights (2012-2016)116 Earth at NightUrban ", + "\nLucknow, India\n2012 - 2016 Change\nLucknow, India\nlights out no change new lights\nChange in Nightlights (2012-2016)116 Earth at NightUrban DevelopmentOlympics at Night\u2014Sochi, Russia\nThree months after an out-of-this-world hand-\noff of the event-opening Olympic torch outside their \norbiting home, the astronauts and cosmonauts on the ISS got to look down on that flame from above. On the evening of February 10, 2014, an ISS Expedition 38 crew member captured this digital photograph of Sochi, Russia, along the coast of the Black Sea. In the image, the Olympic flame burns in the circular Medals Plaza, ringed in gold and bright white light-ing in the center of the Olympic Park. The oval-shaped Fisht Olympic Stadium is lit in blue and stands near the shore to the south. (Note that, owing to the orbit of the ISS, south is to the right in this image.) The Adler Arena Skating Center and the Iceberg Skating Palace both appear as black rectangles north and east of the Medals Plaza, and the Bolshoy Ice Dome has a pink tint and stands to the west.", + ") The Adler Arena Skating Center and the Iceberg Skating Palace both appear as black rectangles north and east of the Medals Plaza, and the Bolshoy Ice Dome has a pink tint and stands to the west. The A-class European Highway, E97, cuts diagonally through the image as it connects the north and south Black Sea regions.\nFebruary 10, 2014\nSochi Airport\n(Adler) 117Urban Development\nOlympic Park\nSochi Airport\n(Adler)\nE97 highway\nFisht Olympic Stadium\n Adler Arena Skating Center\nIceberg Skating Palace\nBolshoy Ice Dome\nMedal Plaza\nBlack Sea\n118 Earth at NightPower Outages\nPower Outages\nHurricane Maria\nHurricane Maria into the Night\u2014Puerto Rico\nThis composite image shows Hurricane Maria on \nSeptember 20, 2017, as it made landfall near Yabucoa, \nPuerto Rico. Data from the Geostationary Operational \nEnvironmental Satellite 13 (GOES\u201313) acquired at 6:15 a.m. \nlocal time is overlaid on true-color data from the MODIS instruments on the Terra and Aqua satellites and nightlight data from the VIIRS DNB sensor.\nSeptember 20, 2017\n350 km 119\nPower Outages\n120 Earth at NightPower OutagesLights Out\u2014Puerto Rico\nHurricane Maria struck Puerto Rico with devastating force in September 2017.", + "\nSeptember 20, 2017\n350 km 119\nPower Outages\n120 Earth at NightPower OutagesLights Out\u2014Puerto Rico\nHurricane Maria struck Puerto Rico with devastating force in September 2017. \nFlooding, downed trees, and toppled power lines made many roads impassable. Most \nof the electricity grid and telecommunications network were knocked offline, leaving 1.5 million people without power. For many locations power wasn\u2019t restored for weeks and even for up to 11 months in some locations. The long power outages, in part, led to the historic property, economic, and life losses in the storm\u2019s aftermath. While 64 people died from direct storm impacts (i.e., via structural collapse, flying debris, floods, and drownings), an estimated 700 to 8400 excess deaths were associated with long-duration disruptions to essential services.\nThe image grid at right shows the number of days that different neighborhoods in \nPuerto Rico were without power following Hurricane Maria. As shown in the top row, urban centers (e.g., Caguas and Arecibo) recovered much faster than adjacent rural towns (e.", + " As shown in the top row, urban centers (e.g., Caguas and Arecibo) recovered much faster than adjacent rural towns (e.g., Gurabo and Utuado, respectively). The bottom four images show different areas of the San Juan metropolitan region. Even within San Juan, some neighborhoods recovered more quickly than other neighborhoods, based largely on how modern grid connections were and whether neighborhoods were close to roads, hospitals, and schools that needed priority attention.\nThe images are derived from data obtained by the VIIRS DNB on the Suomi NPP \nsatellite to generate a data product suite called Black Marble\u2014see Appendix A \nto learn \nhow this pr\noduct is generated. The Black Marble product suite enables near-real-time, \ndaily monitoring of nightlights globally, in big cities, and in remote and isolated areas that may be difficult to reach. In addition to detecting power outages, the Black Marble product suite can be used to track the speed of recovery (i.e., power restoration) at a 121\nPower Outages\nneighborhood level\u2014a capability that previous nightlight products and imagery did not have, as ", + ", power restoration) at a 121\nPower Outages\nneighborhood level\u2014a capability that previous nightlight products and imagery did not have, as more than half \nof the daily data was corrupted by moonlight.\nTo respond to storms like Maria, disaster responders need data, not only about who has lost access to \nelectricity, but also for how long. These data need to be continuously collected, available in real time, and resolved at the street level, so that decision-makers can understand who has been most affected, what is the capacity of the affected communities to cope, and where to send aid. The high resolution of the Black Marble product suite, when combined with high-resolution demographic data, can help provide such vital information.\nNumber of Days Without Electricity\n\u226430 \u2265150122 Earth at NightPower Outages\nHurricane Matthew\nHurricane Matthew Brushes Florida Coast\u2014Southeast United States\nHurricane Matthew strengthened to a major hurricane on September 30, 2016. \nAfter it made landfall in Haiti on October 4, the hurricane leveled entire villages.", + "Hurricane Matthew Brushes Florida Coast\u2014Southeast United States\nHurricane Matthew strengthened to a major hurricane on September 30, 2016. \nAfter it made landfall in Haiti on October 4, the hurricane leveled entire villages. On \nOctober 7, 2016, the storm\u2019s western eyewall brushed parts of Florida. This natural-color image was acquired at 12:00 noon (local time) on October 7, 2016 by the MODIS instrument on NASA\u2019s Terra satellite. An hour before the image was taken, Matthew was located roughly 35 miles (55 kilometers) northeast of Daytona Beach, Florida, and headed toward the north-northwest at a pace of 13 miles (20 kilometers) per hour. It had top sustained winds of 120 miles (195 kilometers) per hour, with higher gusts. 123Power Outages\nOctober 7, 2016\nFloridaGeorgiaSouth CarolinaNorth CarolinaVirginiaWest \nVirginia\nMaryland\nPennsylvania\nDelaware\n200 km124 Earth at NightPower Outages\nLights Out After Matthew\u2014Southeast United States\nAfter grazing Florida and Georgia, Hurricane Matthew plowed into South Carolina, \nsoutheast of McClellanville, as a Category 1 storm. Strong winds, falling trees, and \nstorm surge flooding knocked out power in coastal areas of all three states.", + " Strong winds, falling trees, and \nstorm surge flooding knocked out power in coastal areas of all three states. From space, the outages were clearly visible\u2014and especially at night. The VIIRS DNB on the Suomi NPP satellite captured these three nighttime images of the Atlantic coast. The image on the top left was acquired at 3:14 a.m. Eastern Daylight Time on October 6, 2016; the top middle image shows the same area at 3:14 a.m. on October 7; the top right image was acquired at 2:14 a.m. on October 8. Infrared observations collected by the GOES East satellite were layered on the VIIRS data to make the clouds associated with Matthew more visible. Notice how many cities and towns on the eastern coast of Florida lost power on October 7. In particular, Flagler County, Florida, and Calhoun County, South Carolina, suffered many outages. The map (bottom, right) is based on data from power companies on October 8, 2016. 125Power Outages\nOctober 6, 2016\n350 km\nOctober 7, 2016\n October 8, 2016\nCalhoun County, SC\nOut of power: 98% \nCharleston County, SC\nOut of power: 26%\nHurricane Matthew8:00 a.", + " 125Power Outages\nOctober 6, 2016\n350 km\nOctober 7, 2016\n October 8, 2016\nCalhoun County, SC\nOut of power: 98% \nCharleston County, SC\nOut of power: 26%\nHurricane Matthew8:00 a.m. EDT, October 8\nFlagler County, FLOut of power: 88%\nCustomers Without Power (as reported 9:30 a.m. EDT, October 8)\n20% 40% 60% 80%126 Earth at NightPower OutagesHurricane Michael\nFlorida Slammed by Hurricane Michael\u2014Florida Panhandle\nAt approximately 1:30 p.m. Eastern Daylight Time on October 10, 2018, Hurricane \nMichael made landfall near Mexico Beach, Florida. Wind speeds were estimated to be \n155 miles (250 kilometers) per hour, which made the Category 4 hurricane the strongest \non record to hit the Florida Panhandle. In April 2019 the National Hurricane Center \nannounced that it had reclassified Michael as a Category 5 storm with 160-mile (257-kilometer)-per-hour winds. The storm destroyed homes and knocked out electric power in the area. The Geostationary Operational Environmental Satellite 16 (GOES\u201316) acquired data for the composite image below around 1:00 p.m. Eastern Daylight Time on October 10.", + " The Geostationary Operational Environmental Satellite 16 (GOES\u201316) acquired data for the composite image below around 1:00 p.m. Eastern Daylight Time on October 10. GOES\u201316 data were overlaid on a MODIS Blue Marble image of Earth.\nOctober 10, 2018\nFloridaGeorgiaAlabama\n100 km 127Power Outages\nMichael Churns at Night\u2014Florida Panhandle\nHurricane Michael weakened throughout October 10 and was downgraded to a \ntropical storm by October 11. The VIIRS DNB on the Suomi NPP satellite acquired data \nfor the composite image below in the early morning hours of October 11, as the storm passed over Georgia and South Carolina. The false-color image shows infrared signals from VIIRS known as brightness temperature, which helps distinguish the shape and temperature of the clouds. The image was overlaid on data from the VIIRS DNB.\nOctober 11, 2018\nFloridaAlabama\nGeorgiaSouth \nCarolinaTennesseeVirginia\nNorth \nCarolinaKentucky\n150 km128 Earth at NightPower OutagesLights Out in Michael\u2019s Wake\u2014Florida Panhandle\nOctober 6, 2018\n light cloud cover\nGulf of Mexico\nAlabama\nGeorgia\nFloridaPanama City\n100 km\nOctober 12, 2018\nHurricane Michael\nlight cloud cover\nAlabama\n", + "\nCarolinaTennesseeVirginia\nNorth \nCarolinaKentucky\n150 km128 Earth at NightPower OutagesLights Out in Michael\u2019s Wake\u2014Florida Panhandle\nOctober 6, 2018\n light cloud cover\nGulf of Mexico\nAlabama\nGeorgia\nFloridaPanama City\n100 km\nOctober 12, 2018\nHurricane Michael\nlight cloud cover\nAlabama\nGeorgia\nFlorida\nGulf of Mexico\n100 kmAfter making landfall as a Category 5 storm on October 10, 2018, Hurricane Michael \nknocked out power for at least 2.5 million customers in the southeastern United States, \naccording to the Edison Electric Institute. These images of nighttime lights in Florida, Georgia, and Alabama come from the Suomi NPP satellite and were acquired on \nOctober 6 and October 12, 2018. The first set of images (below) shows a natural view 129Power Outages\nof nightlights from the VIIRS DNB. The second pair of images is a data visualization of \nwhere lights went out in Panama City, Florida. A team of scientists from NASA processed and corrected the raw data to filter out stray light from the Moon, fires, airglow, and any other sources that are not electric lights. Their processing techniques also removed other atmospheric interference, such as dust, haze, and thin clouds.", + " Their processing techniques also removed other atmospheric interference, such as dust, haze, and thin clouds. \nOctober 6, 2018\nPanama City\nSaint Andrew Bay\n1 km\nPanama City\nOctober 12, 2018\n1 km\nSaint Andrew Bay\nNighttime Lights\nLess More130 Earth at NightPower Outages\nHurricane Sandy\nOvernight View of Hurricane Sandy\u2014Eastern United States\nThis image of Hurricane Sandy was acquired by the VIIRS DNB on the Suomi NPP \nsatellite at 2:42 a.m. Eastern Daylight Time on October 28, 2012. Cloud tops were lit \nby the nearly full Moon (full occurred on October 29). Some city lights in Florida and Georgia are visible through the clouds. At the time of the image, the United States National Hurricane Center estimated Sandy\u2019s location to be 31.5\u00b0 North and 73.7\u00b0 West, 275 miles (443 kilometers) south-southeast of Cape Hatteras, North Carolina, and moving northeast at 14 miles (22 kilometers) per hour. Maximum sustained winds were 75 miles (120 kilometers) per hour, and the minimum central barometric pressure was 960 millibars (28.", + " Maximum sustained winds were 75 miles (120 kilometers) per hour, and the minimum central barometric pressure was 960 millibars (28.35 inches). 131Power Outages\nOctober 28, 2012\n250 km132 Earth at NightPower OutagesBlackout\u2014New Jersey and New York\nThis pair of images shows New Jersey, New York, and eastern Pennsylvania as \nviewed at night by the VIIRS DNB on the Suomi NPP satellite. The \u201cbefore\u201d image \n(below) was taken at 2:14 a.m. Eastern Daylight Time on August 31, 2012, when \nconditions in the area were normal. The \u201cafter\u201d image (right) was taken at 2:52 a.m. Eastern Daylight Time on November 1, 2012, after Hurricane Sandy had moved through. In the \u201cafter\u201d image, lingering clouds from Sandy are lit by moonlight and obscure much of New York\u2019s Hudson Valley, northwestern New Jersey, and northeastern \nAugust 31, 2012\nRockaway BeachManhattan\nPhiladelphia\nAtlantic CityMantoloking\n25 km 133\nPower Outages\nPennsylvania. In Manhattan, the lower third of the island is dark on November 1, \nwhile Rockaway Beach, much of Long Island, and nearly all of central New Jersey are significantly dimmer.", + " In Manhattan, the lower third of the island is dark on November 1, \nwhile Rockaway Beach, much of Long Island, and nearly all of central New Jersey are significantly dimmer. The barrier islands along the New Jersey coast, which are heavily developed with tourist businesses and year-round residents, are just barely visible in moonlight after the blackout. Along with the scattered electric lights, there is a bright point along the shore south of Mantoloking, New Jersey, that could be fires fueled by severed natural gas lines.\nNovember 1, 2012\nRockaway Beach\nAtlantic CityPhiladelphiaMantolokingManhattan\nbright point\n25 km134 Earth at NightPower OutagesOther Power Outages\nLights Out in Hatteras\u2014North Carolina\nJuly 27, 2017\nWilmingtonAtlantic Ocean\nHatterasOcracokePortsmouth\n50 kmIf a power outage spans a large area, there is the chance it can be seen from \nspace. The outage pictured here shows the darkness that fell across parts of North \nCarolina\u2019s Outer Banks in late July 2017. The VIIRS DNB on the Suomi NPP satellite captured nighttime images of Hatteras and Ocracoke islands.", + " The VIIRS DNB on the Suomi NPP satellite captured nighttime images of Hatteras and Ocracoke islands. They show the barrier 135\nPower Outages\nislands on July 27 (before the blackout) and on July 30, 2017. Storms are a common \nculprit for power outages, but the power outage pictured here had a more unusual cause\u2014a construction accident that compromised buried transmission cables.\nJuly 30, 2017\nWilmingtonAtlantic OceanPortsmouth\nHatterasOcracoke\n50 km136 Earth at NightPower OutagesRare Derecho Causes Power Outages\u2014Washington, D.C.\nJune 28, 2012\nBaltimore\nWashington, D.C.\nRichmond\nPittsburgh\nroute 270 \u2014\nroute 66 \u2014\nroute 267 \u2014\n50 kmThese before-and-after images from the Suomi NPP VIIRS DNB show the severe \nimpact that a rare, fast-moving thunderstorm system had on the Baltimore, Maryland\u2013 \nWashington, D.C. metropolitan area on June 29, 2012. The storm raced hundreds of miles from west of Chicago across Illinois, Indiana, Ohio, West Virginia, Virginia, \nMaryland, and Washington, D.C.; it combined intense lightning and rain with hurri-cane-force winds with speeds that were upwards of 60 ", + "; it combined intense lightning and rain with hurri-cane-force winds with speeds that were upwards of 60 miles (96 kilometers) per hour. It killed 22 people and caused some 4.3 million households to lose power for days. A 137\nPower Outages\nquick comparison between the June 28 and June 30 images reveals extensive power \noutages around Washington, D.C. and Baltimore. Clouds obscure the lights of Phila-delphia and other areas north and east of Baltimore. Of particular interest is the loss of light to the north and west of Washington, D.C., along interstate highway Routes 270 and 66 and Maryland Route 267. Known as a \u201cderecho,\u201d which is the Spanish word for \u201cstraight,\u201d the forceful winds can be as powerful as tornados, but the winds don\u2019t twist, instead driving in a straight line over long distances; hence the name.\nJune 30, 2012\nBaltimore\nclouds\nWashington, D.C.\nRichmond\nPittsburgh\nroute 270 \u2014\nroute 66 \u2014\nroute 267 \u2014\n50 km138 Earth at NightPower OutagesEarthquake Aftermath\u2014Nepal\nEven before a powerful magnitude 7.", + "\nRichmond\nPittsburgh\nroute 270 \u2014\nroute 66 \u2014\nroute 267 \u2014\n50 km138 Earth at NightPower OutagesEarthquake Aftermath\u2014Nepal\nEven before a powerful magnitude 7.8 earthquake jolted Nepal on April 25, 2015, \ncontinuous access to electricity was not something the Nepalese could take for \ngranted. The country\u2019s rugged topography prevents Nepal\u2019s main electricity provider\u2014the state-owned Nepal Electricity Authority\u2014from providing service to many rural areas. And even in towns and cities where service is available, chronic power shortages mean the 30 million people of Nepal often face lengthy outages. In the aftermath of the earthquake, however, access to reliable electricity diminished even further. As shown by this map, made using VIIRS DNB data from the Suomi NPP satellite, both urban and rural areas face widespread outages after the earthquake. It compares the pre-earthquake period, using data from clear days between March 21\u201330, 2015, and the post-earthquake period, which includes observations from April 19\u201328, 2015. Combining data from several days makes the observations more meaningful and less prone to error.", + "March 21\u201330, 2015, and the post-earthquake period, which includes observations from April 19\u201328, 2015. Combining data from several days makes the observations more meaningful and less prone to error. The map shows how the amount of light emitted by towns and cities in Nepal changed before and after the earthquake. Areas with less light output after the earthquake are shown in shades of orange; areas with the same output are black; areas with more light are purple. The satellite detected widespread power losses after the earthquake. The cities of Kathmandu, Bharatpur, and Hetauda were particularly hard hit. In rural areas, each pixel in the map appears to correspond with a different town or village. 139\nPower Outages\nChina\nKathmandu\nHetauda\nBharatpur\nNepal\nmissing data\n100 km\nLighting Change\nLess Same More140 Earth at NightEffects of WarEffects of War\nConflict in the Middle East\u2014Syria\nSix years of war in Syria have had a devastating effect on millions of its people. \nOne of the most catastrophic impacts has been on the country\u2019s electricity network. The \nleft and middle images (below) were created using two separate nightlight datasets from the VIIRS DNB on ", + " The \nleft and middle images (below) were created using two separate nightlight datasets from the VIIRS DNB on the Suomi NPP satellite for 2012 and 2016, respectively. The image on the far right (below) shows the change in lighting intensity when the 2012 \n2012\nAleppo, Syria\n2016 141\nEffects of War\nand 2016 images are combined. Orange shades indicate areas where lights have \ngone out over the years, leaving people to survive with little or no power. Syria was once one of the eastern Mediterranean\u2019s major power suppliers. It has 15 power sta-tions, including 3 major hydroelectric dams. To view global changes in lighting inten-\nsity from 2012 to 2016, see \npage 26 of this volume.\nAleppo, Syria\n2012 - 2016 Change\nAleppo, Syria\nChange in Nightlights (2012-2016)\nlights out no change new lights142 Earth at NightEffects of WarNightlights Change in the Middle East\u2014Syria and Iraq\nThe images below show differences in nighttime lighting between 2012 (below) \nand 2016 (right) in Syria and Iraq, among several Middle Eastern countries.", + "Change in the Middle East\u2014Syria and Iraq\nThe images below show differences in nighttime lighting between 2012 (below) \nand 2016 (right) in Syria and Iraq, among several Middle Eastern countries. Each \nimage is drawn from a global composite that was made by selecting the best cloud-free nights in each month over each land mass. The data come from the VIIRS DNB on the Suomi NPP satellite. The changes are most dramatic around Aleppo, but also \nextend through western Syria to Damascus. Over the four years shown, lighting increased in areas north of the Syrian border in Turkey and to the west in Lebanon. In Iraq, some northern sections near Mosul saw a decrease in light over the years, while areas \n2012\nSyria\nAleppo\nDamascus\nEuphrates River\nIraq\nMosul\nTigris River\nBaghdad\nIrbil\nKirkuk\nTurkey\n100 km 143Effects of War\naround Baghdad, Irbil, and Kirkuk saw increases. Note, too, the change in electric \nlight patterns along the Tigris and Euphrates river basins. International agencies such as the United Nations Institute for Training and Research Operational Satellite \nApplications Programme (UNITAR-UNOSAT) have used such imagery in the past few years to support United Nations ", + " International agencies such as the United Nations Institute for Training and Research Operational Satellite \nApplications Programme (UNITAR-UNOSAT) have used such imagery in the past few years to support United Nations operations and activities in the areas of disaster \nresponse, humanitarian support, human security, and human rights. Nighttime \nimagery helps relief and peacekeeping groups identify areas that are most in need of aid and support.\n2016\nSyria\nAleppo\nDamascus\nEuphrates River\nIraq\nMosul\nTigris River\nBaghdad\nIrbil\nKirkuk\nTurkey\n100 km144 Earth at NightMiningMining\nShale Revolution: As Clear as Night and Day\u2014South Texas\n\u201cPlay\u201d is a term used by petroleum geologists to describe a geological formation \nthat has been targeted for exploration because it likely contains oil or gas. In nighttime \nsatellite imagery, the light from the Eagle Ford Shale Play competes with the nearby \ncities of San Antonio and Austin, Texas. The electric glow of drilling equipment, worker camps, and other gas and oil infrastructure combine with flickering gas flares to create an unmistakable arc of light across southern Texas.", + " The electric glow of drilling equipment, worker camps, and other gas and oil infrastructure combine with flickering gas flares to create an unmistakable arc of light across southern Texas. On February 15, 2016, the VIIRS DNB on the Suomi NPP satellite captured this nighttime view of Eagle Ford (below). The Eagle Ford Shale, which is about 400 miles (600 kilometers) long and 50 miles (80 kilometers) wide, is a source of both oil and gas. Most of the oil-producing wells are located on the northern part of the play; the gas-producing wells are located along its southern edge. As shown in the two images (right) of Cotulla (outlined \nFebruary 15, 2016\nMexico\nTexas\n San Antonio\nAustin\n|\nCotulla outline\nEagle Ford Shale Play\n50 km 145Miningin white in the VIIRS DNB image, left), the view is also stunning during daylight. In the early 2000s, the area east \nof Cotulla, Texas, was dry, sleepy shrubland. By 2015 a bustling network of roads and rectangular drill pads had completely transformed the landscape. The pair of satellite images shows how much the landscape has changed. Landsat 5 acquired the December 17, 2000, image; Landsat 8 captured the December 18, 2015, image.", + " The pair of satellite images shows how much the landscape has changed. Landsat 5 acquired the December 17, 2000, image; Landsat 8 captured the December 18, 2015, image. Accord-ing to a report from the Texas Observer, a nonprofit news organization based in Austin, Cotulla saw its population swell from about 4,000 to 10,000 people in just a few years due to an influx of oil and gas workers.\nDecember 17, 2000\nCotulla\n5 km\nDecember 18, 2015\nCotulla\n5 km146 Earth at NightMining\nTen Percent of the World\u2019s Gas Flares in One Spot\u2014Nigeria\nAt night, gas flares\u2014used to remove unwanted natural gas found in crude oil\u2014\noutshine everything else in the Niger River Delta. In this image of Nigeria taken on \nDecember 18, 2013, by the VIIRS DNB on the Suomi NPP satellite (top right), the lights of Port Harcourt and Benin City are dim compared to the flares. The image illustrates two \nfacts from a U.S. Energy Information Administration assessment: Nigeria contains more gas flares than any other country except Russia, and Nigeria has one of the lowest per capita electricity generation rates in the world.", + "Information Administration assessment: Nigeria contains more gas flares than any other country except Russia, and Nigeria has one of the lowest per capita electricity generation rates in the world. About ten percent of the world\u2019s gas flares are located in Nigeria, and most of them are concentrated in the Delta region. The flares and oil production occur both on land and offshore. It is hard to see where the land ends and the ocean begins in the nightlights image, which shows the Delta region in visible light as it might appear to the human eye. But viewing the scene in infrared light reveals the distribution of the flares.\nThe VIIRS image (bottom right) shows the same area on the same night in midwave \ninfrared light, a portion of the electromagnetic spectrum often used to study emitted thermal radiation at night. In this view, warm ocean waters are brighter than the cool land and cold clouds, making it possible to see the boundary between land and water. The flares shine brightly in both views. 147Mining\nDecember 18, 2013\ncity lights (Benin City)\ncity lights (Port Harcourt)\ncloud\ngas flares\n50 km\nDecember 18, 2013\nland\nocean\ncloud\nNiger River\ngas flares\n50 km148 Earth at ", + " 147Mining\nDecember 18, 2013\ncity lights (Benin City)\ncity lights (Port Harcourt)\ncloud\ngas flares\n50 km\nDecember 18, 2013\nland\nocean\ncloud\nNiger River\ngas flares\n50 km148 Earth at NightMiningGas Flares in Bah\u00eda de Campeche\u2014Gulf of Mexico\nIn the early 1970s, oil exploration turned up vast reservoirs of oil and gas under the \nBahia de Campeche (Bay of Campeche), located along the southern margin of the Gulf \nof Mexico, just west of Mexico\u2019s Yucat\u00e1n Peninsula. Offshore oil drilling continues in that region today, and signs of the activity are visible from space as crude oil often contains natural gas, which is disposed of by flaring, as noted earlier. On September 13, 2009, the Advanced Land Imager (ALI) on NASA\u2019s Earth Observing-1 (EO-1) satellite captured a natural-color image of gas flares and an oil slick in the Bahia de Campeche (below). \nSeptember 13, 2009\noil slick\ngas flares\ngas flare\n1 km 149Mining\nSunglint\u2014sunlight reflecting off the ocean surface and back to the satellite\u2014gives the \nocean a silver-gray appearance and also illuminates the oil slick, which smoothed the ocean surface.", + " \nSeptember 13, 2009\noil slick\ngas flares\ngas flare\n1 km 149Mining\nSunglint\u2014sunlight reflecting off the ocean surface and back to the satellite\u2014gives the \nocean a silver-gray appearance and also illuminates the oil slick, which smoothed the ocean surface. Nearly three years later (July 26, 2012), the VIIRS DNB on the Suomi NPP satellite captured a nighttime image (below) of city lights and oil production. Gas flares appear as extremely bright central spots surrounded by a circular halo. Electric lights in cities and oil production sites vary in brightness, and do not have a halo.\nJuly 26, 2012\nGulf of Mexico\nYucatan Peninsula\noffshore oil production\nonshore oil production\n50 km150 Earth at NightMining\nGas Drilling\u2014North Dakota\nNorthwestern North Dakota is one of the \nleast-densely populated parts of the United \nStates. Cities and people are scarce, but \nsatellite imagery shows the area has been aglow at night in recent years. The reason: the area is home to the Bakken shale formation, a site where gas and oil production are booming. On November 12, 2012, the VIIRS DNB on the Suomi NPP satellite captured this nighttime view of widespread drilling throughout the area.", + " On November 12, 2012, the VIIRS DNB on the Suomi NPP satellite captured this nighttime view of widespread drilling throughout the area. Most of the bright specks are lights associated with drilling equipment and temporary housing near drilling sites, although a few are evidence \nof gas flaring. Some of the brighter areas \ncorrespond to towns and cities including \nWilliston, Minot, and Dickinson. When VIIRS acquired the image, the Moon was in its waning crescent phase, so the landscape reflected only a small amount of its light.\nNovember 12, 2012\nSaskatchewan\nWilliston\nMontana\n50 km 151Mining\nManitoba\nWilliston\nNorth Dakota\nDickinson\nMinotBakken Formation152 Earth at NightMining\nConnection Between Gas Flaring and Arctic Pollution\u2014North Dakota\nPrevious research has suggested that gas flares from oil and natural gas \nextraction in the Northern Hemisphere could be a key source of black carbon \npollution in the Arctic. But since international inventories of industrial emissions have gaps in observations and reporting, they often over- or underestimate the amount of pollutants.", + " But since international inventories of industrial emissions have gaps in observations and reporting, they often over- or underestimate the amount of pollutants. Gas flares are an often-overlooked subset in that incomplete dataset. Data from the VIIRS DNB on the Suomi NPP satellite were used to examine gas flare signals from nightlights (bottom right) and the nitrogen dioxide retrievals (top right) for four regions around the planet; only the Bakken Formation in North Dakota is shown here. Levels of atmospheric nitrogen dioxide were found to rise about 1.5 percent per year at Bakken. This means the concentration of black carbon produced by those flares was also likely on the rise. Such local or regional nightlight data as are described here clearly show the potential for global consequences. 153Mining\n2005 - 2014\nNorth Dakota\nSouth DakotaSaskatchewan\nMontana\nMinnesotaManitobaNorthwest \nOntario\nBakken Formation\nN100 km\nNO2 Column Density Change, 2005-2014 (x1014 molecules/cm2)\n\u22640 3.75 \u22657.5\n2014 - 2015\nNorth Dakota\nSouth DakotaMinnesotaSaskatchewan\nMontanaManitobaNorthwest \nOntario\nBakken Formation\n100 km154 Earth at NightSea-Going ", + "5\n2014 - 2015\nNorth Dakota\nSouth DakotaMinnesotaSaskatchewan\nMontanaManitobaNorthwest \nOntario\nBakken Formation\n100 km154 Earth at NightSea-Going VesselsSea-Going Vessels\nSomething Fishy in the Atlantic Night\u2014South Atlantic Ocean\nFirst noted in the late 1970s and early 1980s, about 200 to 300 miles (322 to 483 \nkilometers) off the coast of Argentina, a city of light routinely appears in the middle of \nthe South Atlantic Ocean as shown in the 2012 composite image below, created with \ndata from the VIIRS sensor on the Suomi NPP satellite. There are no human settlements there, nor fires or gas wells. But there are lots of fishing boats. Squid fishermen adorn their boats with bright lights for night fishing to draw prey into their nets. The boats cluster offshore along invisible lines: the underwater edge of the continental shelf, the nutrient-rich Malvinas Current, and the boundaries of the exclusive economic zones of Argentina and the Falkland Islands. The maps on the right show the locations of fishing boats on nine consecutive nights from April 17 to 25, \n2012, also obtained with VIIRS on Suomi NPP .", + " The maps on the right show the locations of fishing boats on nine consecutive nights from April 17 to 25, \n2012, also obtained with VIIRS on Suomi NPP . Note that lights appear sharper on some nights and more diffuse on others due to the presence or absence of cloud cover and fog. While not shown specifically here, other vessels are involved: In addition to the fishing boats, large refrigeration and refu-eling ships keep the long-distance operators working without having to go back to a port. Satellite images like these allow scientists to better understand and manage fisheries in international waters; for example, they can estimate the weekly captures of different species.\n2012\nSouth America\nAtlantic Ocean\nBah\u00eda Blanca\nComodoro Rivadavia\n200 km 155\nSea-Going Vessels\nApril 17, 2012 April 18, 2012 April 19, 2012\nApril 20, 2012 April 21, 2012 April 22, 2012\nApril 23, 2012 April 24, 2012 April 25, 2012\n50 km156 Earth at NightSea-Going Vessels\nKorea and the Yellow Sea\u2014Korean Peninsula\nOn September 24, 2012, the ", + "April 20, 2012 April 21, 2012 April 22, 2012\nApril 23, 2012 April 24, 2012 April 25, 2012\n50 km156 Earth at NightSea-Going Vessels\nKorea and the Yellow Sea\u2014Korean Peninsula\nOn September 24, 2012, the VIIRS DNB on the Suomi NPP satellite captured \nthis nighttime view of the Korean Peninsula. The wide-area image shows the \nKorean Peninsula, parts of China and Japan, the Yellow Sea, and the Sea of Japan \n(also known as the East Sea). The white inset box, magnified in the image on the \nbottom right, shows the lights of fishing boats in the Yellow Sea; many of the boats appear to form a line, as if marking a watery boundary between nations. 157Sea-Going Vessels\nYellow Sea\nChinaSeptember 24, 2012\nSea of Japan\nSouth\nKorea\nSeoul\nNorth\nKorea\nPyongyang\n200 km\n20 km158 Earth at NightHoliday LightsHoliday Lights\nBursting with Holiday Energy\u2014United States\nNASA researchers found that nighttime lights in the United States shine 20 to 50 \npercent brighter in December due to holiday light displays and other activities during \nChristmas and New Year\u2019s when compared to light output during the rest of the year.", + "found that nighttime lights in the United States shine 20 to 50 \npercent brighter in December due to holiday light displays and other activities during \nChristmas and New Year\u2019s when compared to light output during the rest of the year. \nThe next five maps (see also pages 161\u2013163), created using data from the VIIRS DNB \non the Suomi NPP satellite, show changes in lighting intensity and location ar\nound \nmany major cities, comparing the nighttime light signals from December 2012 and \n2012 \u2013 2014\nWashington, D.C.\nVirginia\nRichmond\nNorfolk\nNorth Carolina\nRaleigh\nWest \nVirginiaMaryland\nOhio\nTennesseeKentucky\nDelaware\nNew \nJersey\nSouth Carolina\nAtlantic \nOcean\n100 km\nHoliday Lighting\nless no change more159\nHoliday Lights\n2013 to the average light output for the rest of 2012 to 2014. On any given night, the \nsignal is subtle; however, averaged over days and weeks, it is more perceptible. Green shading marks areas where light usage increased in December; yellow marks areas with little change; and red marks areas where less light was used.\nThe light output from 70 U.S. cities was examined as a first step towar\nd determining \npatterns in urban energy use.", + "\nThe light output from 70 U.S. cities was examined as a first step towar\nd determining \npatterns in urban energy use. They found that light intensity increased by 30 to 50 \n2012 \u2013 2014\nGeorgia\nAtlanta\nNorth Carolina\nCharlotte\nSouth Carolina\nColumbia\nAlabama\nBirmingham\nTennessee\nNashville\nAtlantic \nOcean\n100 km\nHoliday Lighting\nless no change more160 Earth at NightHoliday Lights\npercent in the suburbs and outskirts of major cities where there is more yard space \nand more single-family homes. Lights in the central urban areas did not increase as \nmuch as in the suburbs but still brightened by 20 to 30 percent. Despite being ethnically and religiously diverse, the U.S. experiences a holiday increase across most urban communities\u2014tracking a national, shared tradition.\nWhile the trend says some interesting things about culture, it also tells us something \nimportant about energy use. A daily global dynamic dataset of nighttime lights provides a new way to understand the broad societal forces impacting energy decisions.", + " A daily global dynamic dataset of nighttime lights provides a new way to understand the broad societal forces impacting energy decisions. As noted by the Intergovernmental Panel on Climate Change, improvements in energy efficiency and conservation are critical for making greenhouse gas reductions. Daily nightlight data provide a new way of looking at how people use cities and the forces and patterns driving energy use.\n(images continue on pages 161\u2013163) 161\nHoliday Lights\n2012 \u2013 2014\nFlorida\nMiami\nOrlando\nJacksonville\nTampa Bay\nGulf of Mexico\n100 km\nHoliday Lighting\nless no change more162 Earth at NightHoliday Lights(cont\u2019d from previous page)\n2012 \u2013 2014\nTexas\nHouston\nAustin\nSan Antonio\nDallas\nLouisiana\nShreveport\nArkansas\n50 km\nHoliday Lighting\nless no change more 163\nHoliday Lights\n2012 \u2013 2014\nArizona\nPhoenix\nNevada\nLas Vegas\nCalifornia\nLos Angeles\nSan Diego\nMexico\nTijuana\n50 km\nHoliday Lighting\nless no change more164 Earth at NightHoliday LightsThe Lights of Ramadan and Eid al-Fitr\u2014Middle East\nIn December 2014, NASA researchers announced ", + "50 km\nHoliday Lighting\nless no change more164 Earth at NightHoliday LightsThe Lights of Ramadan and Eid al-Fitr\u2014Middle East\nIn December 2014, NASA researchers announced that they had detected significant \nchanges in the amount and distribution of nighttime lighting during the holy month of \nRamadan in the Middle East.\nThe maps on pages 165 and 166 show changes in lighting intensity and location \non the Arabian Peninsula and in the countries along the eastern Mediterranean coast. They are based on data from the VIIRS DNB on the Suomi NPP satellite. These maps compare the nightlight signals from the months of Ramadan in 2012\u20132014 (parts of July and August in those years) to the average light output for the rest of 2012 to 2014. Green shading marks areas where light usage increased during the holy days; yellow marks areas with little change; and red marks areas where less light was used.\nA large increase in light output around the Egyptian capital Cairo corresponded \nwith the holy month of Ramadan. The change made sense because Muslims fast during daylight in Ramadan, pushing meals, social gatherings, commerce, and other activities into nighttime hours.", + " The change made sense because Muslims fast during daylight in Ramadan, pushing meals, social gatherings, commerce, and other activities into nighttime hours. Peaks in light use closely tracked the Islamic calendar, as Ramadan shifted earlier in the summer each year.\n(cont\u2019d on page 167) 165\nHoliday Lights\n2012 \u2013 2014\nEgypt\nCairo\nIsrael\nTel Aviv\nJerusalem\nJordan\nAmman\nSyria\nDamascus\nBeirut\nLebanon\nMediterranean Sea\n100 km\nHoliday Lighting\nless no change more166 Earth at NightHoliday Lights\n2012 \u2013 2014\nSaudi \nArabia\nRed \nSea\nRiyadh\nJeddah\nMecca\nMedina\n100 km\nHoliday Lighting\nless no change more 167\nHoliday Lights\n(cont\u2019d from page 164)\nLight use in Saudi Arabian cities, such as Riyadh and Jeddah, increased by 60 to \n100 percent throughout the month of Ramadan. Light use in Turkish cities, however, \nincreased far less. Some regions in Syria, Iraq, and Lebanon did not have an increase in light output\u2014and some even demonstrated a moderate decrease, possibly due to unstable electrical grids and conflict in the region.", + " Some regions in Syria, Iraq, and Lebanon did not have an increase in light output\u2014and some even demonstrated a moderate decrease, possibly due to unstable electrical grids and conflict in the region. Even within majority Muslim popu-lations, there are a lot of variations. Lighting patterns track cultural variation within the Middle East.\nThese variations appear even at the neighborhood level. Some of the poorest and \nmost devout areas observed Ramadan without significant increases in light use throughout the month, choosing\u2014whether for cultural or financial reasons\u2014to leave their lights off at night.\nHowever, during the Eid al-Fitr celebration that marks the end of Ramadan, light \nuse soared across all study groups, as all the neighborhoods appeared to join in the festivities. Energy use patterns seem to reflect social and cultural identities, as well as the habits of city dwellers, and not just price or other commercial factors.168 Earth at NightEpilogue\nOver the last 50 years of the space program we have had the opportunity to \nview planet Earth not only during the day but also at night.", + "168 Earth at NightEpilogue\nOver the last 50 years of the space program we have had the opportunity to \nview planet Earth not only during the day but also at night. As so many \nastronauts have remarked, seeing the day-lit Earth from space changes our per -\nspective. We move away from feeling like local or regional creatures, living in \ncountries, states, and towns, towards a new perspective of becoming citizens of \nthe whole world. This shift in awareness is because the day-lit Earth shows no \nartificial boundaries created by humans to separate different societies and political \nsystems. But this perspective changes at night, when we see the places on Earth \nwhere progress has produced vivid displays of lights.\nEarth evolves through a process of dynamical change. This evolution surely \nincludes human inhabitants, since we live, work, multiply, and migrate within Earth\u2019s \nbiosphere on a continual basis. Throughout human history on Earth, civilizations have \ncome and gone due to natural disasters, long-term climate change, and the con -\nsequences of large meteorite impacts.", + " Throughout human history on Earth, civilizations have \ncome and gone due to natural disasters, long-term climate change, and the con -\nsequences of large meteorite impacts.\nWhat will Earth at night look like from space in the next 50 years? Will there \nbe new and sprawling cities where today there are none? Will natural disasters \ndestroy densely populated areas we see today, turning them into dark, forbidden \nareas at night? Surely all these things will happen to some extent. 169\nChances are Earth at night will only get brighter with more complex webs of \nlights strung all over the planet. This shows progress and is important for sustaining \nlife, but my hope is that there will still be places where the darkness is treasured, \nand where we can continue to look outward to the planets, stars, and beyond.\nJames L. Green\nNASA Chief Scientist170 Earth at NightAppendix A\nNASA\u2019s Black Marble Product Suite\nNASA\u2019s Black Marble product suite provides estimates of daily nighttime lights and \nother intrinsic surface optical properties of Earth at night. The product is based on the \nDay/Night Band (DNB) sensor of the Visible Infrared Imaging Radiometer Suite (VIIRS)", + " The product is based on the \nDay/Night Band (DNB) sensor of the Visible Infrared Imaging Radiometer Suite (VIIRS) \ninstrument onboard the Suomi National Polar-orbiting Partnership (NPP) satellite. The \nVIIRS DNB is a highly sensitive, low-light sensor capable of measuring daily global \nnighttime light emissions and reflections, allowing users to identify sources and intensi -\nties of these artificial lights, and monitor changes over a period of time. Like any opti -\ncal sensor, the principal challenge in using VIIRS DNB data is to account for variations \nin light captured by the sensor. Certainly, there are variations in the sources and inten -\nsity of anthropogenic light due to changing human processes like urbanization, oil and \ngas production, nighttime commercial fishing, and infrastructure development. However, \nthese processes can only be studied when other naturally-occurring factors that \ninfluence nighttime lights are removed. For example, variations in lunar lighting due to \nconsistent changes in Moon phase cause fluctuations in the amount of light shining \non Earth.", + " For example, variations in lunar lighting due to \nconsistent changes in Moon phase cause fluctuations in the amount of light shining \non Earth. Similarly, land-cover dynamics (e.g., seasonal vegetation, snow and ice \ncover), as well as atmospheric conditions (e.g., clouds, aerosols, airglow, and auroras), \ninfluence the intensity of the light captured by the sensor as it travels over various parts of \nthe world.\nTo realize the full potential of the VIIRS DNB time series record for nighttime lights \napplications, NASA\u2019s Black Marble product suite was developed, building on a history \nof 20 years of research on how light changes when it reflects off of surfaces with dif -\nferent angular and spectral properties. Through complex modeling, scientists can now \npredict how moonlight, snow, vegetation, terrain, and clouds impact the lights we 171\nsee. The Black Marble product suite removes these extraneous effects and creates \nconsistent daily records of man-made lighting (see figure above). The product ingests \nmultiple sources of ancillary data to output the highest-quality, pixel-based estimates of nighttime lights.", + " The product ingests \nmultiple sources of ancillary data to output the highest-quality, pixel-based estimates of nighttime lights.\n[Left] Overview of NASA\u2019s \nBlack Marble product \nretrieval process. During the portion of the lunar cycle when moonlight is present at the time of satel-lite observation, the surface upward radiance from artificial light emissions (shown in red) can be extracted from at-sensor radiance (white lines with arrows towards satellite). The Black Marble product suite removes several nat-ural sources that influence the at-sensor radiance, including airglow, moonlight, snow and seasonal vegeta-tion, stray light, and clouds.\nWith these corr\nections in nighttime radiances, we get a superior view of nighttime \nlights with frequent observations and reduction in background noise, enabling various \nquantitative analyses of daily, seasonal, and annual changes, in an analysis-ready file format. Products are calibrated across time, validated against ground measurements, and include quality indicators so that they can be used effectively and accurately in science and applications studies.", + " Products are calibrated across time, validated against ground measurements, and include quality indicators so that they can be used effectively and accurately in science and applications studies.\nThe product is routinely made available from NASA\u2019s Level 1 and Atmosphere \nArchive and Distribution System (LAADS) Distributed Active Archive Center (DAAC), and can be freely accessed through NASA\u2019s EarthData portal: https://urs.earthdata.nasa.gov/home. The standard product is available at 500-meter spatial resolution since 172 Earth at NightJanuary 2012 and is being processed on a daily basis within 3-5 hours after acquisition, \nenabling both near-real-time uses and long-term monitoring applications. Detailed information on product generation, applications, and documentations can be found at the product\u2019s website (https://blackmarble.gsfc.nasa.gov).\nThese products are not just pretty maps of Earth at night. They provide a global \nnighttime, environmental-science-quality product needed for all major disciplines that focus on the nocturnal environment, with emphasis on targeted studies of phenomena related to, for example, light pollution, urbanization science, disaster response, ", + "\nnighttime, environmental-science-quality product needed for all major disciplines that focus on the nocturnal environment, with emphasis on targeted studies of phenomena related to, for example, light pollution, urbanization science, disaster response, clouds and aerosols, and the ocean at night, as well as various applications like energy access, disaster risk reductions, and resilience. These daily records are invaluable to city planners, ecologists, economists, and emergency responders assessing damage after \nmajor storms like Hurricane Maria (discussed on pages 118\u2013121 of this volume). Since \nlight at night is a primarily human-created phenomenon, these products will improve our understanding of human processes that take place on the land and ocean, and interactions between human systems and the environment. 173\nAppendix B\nMaking a Cloud-Free, Global, Earth-at-Night Image \nUsing NASA\u2019s Black Marble Product Suite\nIn its orbit, the Suomi National Polar-orbiting Partnership (NPP) \nsatellite platform flies 512 miles (824 kilometers) above Earth, \ncrossing the equator at roughly 1:30 a.m. and 1:30 p.m. local time, while circling the planet from pole to pole about 14 times a day.", + "\nsatellite platform flies 512 miles (824 kilometers) above Earth, \ncrossing the equator at roughly 1:30 a.m. and 1:30 p.m. local time, while circling the planet from pole to pole about 14 times a day. A key instrument on Suomi NPP is the Visible Infrared Imaging Radiometer Suite (VIIRS), which is a multispectral instrument that scans 1889-mile- (3040-kilometer-) wide swaths of data.1\nVIIRS instrument\nactive scanning swath\n2\nnighttime\nswath\nThe Suomi NPP satellite observes Earth\u2019s surface throughout the day\u2014and at night. The Day/Night Band (DNB) of the \nVIIRS instrument detects light in a range of wavelengths from green (500 nanometers) to near-infrar\ned (900 nanometers) \nand uses filtering techniques to observe dim signals such as city lights\u2014down to the scale of an isolated highway lamp\u2014wildfires, gas flares, auroras, and reflected moonlight at night. In this example (above), the satellite acquired a swath of data during nighttime hours over portions of South America, the Atlantic Ocean, and the Eastern United States.174 Earth at Night3\n1 2 3nighttime swaths\nEvery day, swaths of data\u2014acquired during day and night hours\u2014can be stitched together to create daytime \nan\nd n\nighttime ", + "174 Earth at Night3\n1 2 3nighttime swaths\nEvery day, swaths of data\u2014acquired during day and night hours\u2014can be stitched together to create daytime \nan\nd n\nighttime images of Earth. In the example above, three nighttime swaths show lights at night over much of \nNorth and South America.\n4\nDaily images of Earth at night, available via NASA\u2019s Black Marble product suite (see Appendix A), were then \nstitched together to create the Black Marble global composite image (above). A\nway from the cities, much of the \nlight observed by Suomi NPP during nighttime is from wildfires. In other places, fishing boats, gas flares, oil drilling, \nor mining operations show up as points of light. 175\nThe final step in making an image of Earth at night involves combining a background \nEarth image with the Black Marble global composite image to give context to dark areas. \nAny background image could have been used\u2014in this case, a global true-color image called the Blue Marble: Next Generation, or BMNG, was used, made with data from the Moderate Resolution Imaging Spectroradiometer (MODIS), which flies on NASA\u2019s Terra and Aqua platforms.", + "true-color image called the Blue Marble: Next Generation, or BMNG, was used, made with data from the Moderate Resolution Imaging Spectroradiometer (MODIS), which flies on NASA\u2019s Terra and Aqua platforms. The BMNG was then modified to appear more \u201cnight-like\u201d to highlight Earth\u2019s land surfaces. The land appears in shades of dark blue and the ocean appears black. The Black Marble global composite image was tinted yellow to closely match the actual appearance of lights from space. The night-like background and Black Marble global composite images were then combined to produce this image of Earth at night.\n5\nFinal combination of the night-\nlike background and yellow-tinted Black Marble global composite images.\nBackground image adjusted to appear night-like.\nBackground image - \nBMNG true-color MODIS.176 Earth at NightAppendix C\nOLS and VIIRS Technical Details\nThe beautiful, evocative, and informative images that make up the bulk of this volume \nare the result of advances in technology, science, and computation\u2014the details of \nwhich would form an entire volume of their own. For those who are interested delving deeper, we provide a bit more information, primarily about the technical ", + " For those who are interested delving deeper, we provide a bit more information, primarily about the technical details of the sensors that enable such images.\n[Right] An artist\u2019s concept \ndepicting a DMSP satel-lite, which series includes DMSP-8, DMSP-9, DMSP-10, DMSP-11, DMSP-12, DMSP-13, and DMSP-14.\nCredit: U.S. Airforce\nThe Operational Linescan System (OLS) onboard the Defense Meteorological \nSatellite Program (DMSP) satellites, one of which is depicted above, pr\novides global \ncoverage in both visible (Band 1) and infrared (Band 2) modes. An earlier, popularized \nversion of Earth at night was created using DMSP OLS data from October 1, 1994 to March 31, 1995. DMSP OLS has been a highly successful sensor, but it is dependent \non older technology with lower resolution than scientific research now demands. 177\nThe Visible Infrared Imaging Radiometer Suite (VIIRS) on the Suomi National Polar-\norbiting Partnership (NPP) satellite has 22 channels, with responses ranging from 412 \nto 12,010 nanometers.", + " 177\nThe Visible Infrared Imaging Radiometer Suite (VIIRS) on the Suomi National Polar-\norbiting Partnership (NPP) satellite has 22 channels, with responses ranging from 412 \nto 12,010 nanometers. Five of these channels are high-resolution imagery bands or \u201cI-bands,\u201d as they are commonly referred to, and sixteen are designed as moderate-resolution bands or \u201cM-bands.\u201d One of these M-bands is the Day/Night Band, or DNB, that is a panchromatic band sensitive to visible and near-infrared wavelengths. (See Appendix B \nto learn how the DNB is used to create nighttime imagery.)\n[Left] The range for visible \nlight is about 400 (blue light) to 700 (red light) nanometers, while the VIIRS DNB\u2019s sensitivity is 500 to 900 nanometers and the DMSP OLS\u2019s sensitivity is 400 to 1100 nanometers.\nSpecifically, the range for visible light is about 400 (blue light) to 700 (r\ned light) \nnanometers, while the VIIRS DNB\u2019s sensitivity is from 500 to 900 nanometers. The \nvarious ranges and a comparison of VIIRS\u2019s spectral capabilities with those of OLS is found in the graphic on this page; the table (next page) provides more technical detail about these two instruments.", + " The \nvarious ranges and a comparison of VIIRS\u2019s spectral capabilities with those of OLS is found in the graphic on this page; the table (next page) provides more technical detail about these two instruments. Because of these capabilities, the VIIRS DNB can easily \u201csee\u201d most common light sources such as incandescent and fluorescent light bulbs, 178 Earth at NightTable. Parameters of VIIRS and OLS\nParameter VIIRS OLS\nOrbit Polar Polar\nOverpass time 1:30 p.m. and 1:30 a.m., local 8:30 \u2013 9:30 a.m. and 8:30 \u2013 9:30 p.m., local\nSwath width 3040 km (~1889 mi) 3000 km (~1864 mi)\nTemporal resolution 12 h 12 h\nSpatial resolution 742 m (~2434 ft) 2.7 km (~1.7 mi)\nWavelength range 500 \u2013 900 nanometers 400 \u2013 1,100 nanometers\nas their spectral ranges significantly overlap with the VIIRS DNB spectral range. The \nonly anthropogenic visible light source that is not always easily detected by the VIIRS \nDNB is light from LED bulbs. For example, white LED bulbs peak near 450 nanometers, \nand therefore return dimmer signals. Despite this limitation, VIIRS DNB is highly sensitive \nto very low amounts of light; it can sense light 100,000 times fainter than conven -\ntional visible-light sensors.", + " Despite this limitation, VIIRS DNB is highly sensitive \nto very low amounts of light; it can sense light 100,000 times fainter than conven -\ntional visible-light sensors.\nUnlike cameras that many use every day to capture a scene in one complete expo -\nsure, VIIRS produces an image by repeatedly scanning a scene (increasing exposure \ntime) and resolving it as millions of individual picture elements, or pixels. The DNB \ngoes a step further, determining on the fly whether to use its low-, medium-, or high-\ngain mode. If a pixel is very bright, the low-gain mode on the sensor prevents the pixel \nfrom oversaturating. If the pixel is dark, the signal will be amplified. 179\nAppendix D\nAdditional Credits and Information\nBlack Marble: front and back covers, endpapers, ii, iii, ix, 2, 3, 5, 13, 26, 27, 114, 115, \n120, 121, 128, 129, 139, 140\u2013143, 158, 159, 161\u2013163, 165, 166, 170\u2013172, 174\nNASA\u2019s Earth Observatory: 9, 14\u201325, 30\u201343, 46\u2013119, 122\u2013167\nInternational Space Station (ISS): xvi, 1, 14, 15, 16, 17, 18, 19, 20, 21, 28, 29, 50, 55, \n60\u201366, 72, 73, 78\u201381, 85, 87, 89\u201395, 97, 98, 100\u2013103, 111\u2013113, 116, 117\nFor more information\nBlack Marble\nhttps://blackmarble.", + " 179\nAppendix D\nAdditional Credits and Information\nBlack Marble: front and back covers, endpapers, ii, iii, ix, 2, 3, 5, 13, 26, 27, 114, 115, \n120, 121, 128, 129, 139, 140\u2013143, 158, 159, 161\u2013163, 165, 166, 170\u2013172, 174\nNASA\u2019s Earth Observatory: 9, 14\u201325, 30\u201343, 46\u2013119, 122\u2013167\nInternational Space Station (ISS): xvi, 1, 14, 15, 16, 17, 18, 19, 20, 21, 28, 29, 50, 55, \n60\u201366, 72, 73, 78\u201381, 85, 87, 89\u201395, 97, 98, 100\u2013103, 111\u2013113, 116, 117\nFor more information\nBlack Marble\nhttps://blackmarble.gsfc.nasa.gov\nhttps://earthobservatory.nasa.gov/features/NightLights\nBlack Marble Science Team\nhttps://blackmarble.gsfc.nasa.gov /#people\nNASA\u2019s Earth Observatory\nhttps:// earthobservatory.nasa.gov\nThe Electromagnetic Spectrum\nhttps://science.nasa.gov/ems/01_intro\nhttps://imagine.gsfc.nasa.gov/science/toolbox/emspectrum1.htmlNational Aeronautics and Space Administration\nwww.nasa.gov\nNP-2019-07-2739-HQ" + ], "employee_handbook.pdf": [ "Contoso Electronics \nEmployee Handbook \n \n \n \n \n \n \n \nThis document contains information generated using a language model (Azure OpenAI). The \ninformation contained in this document is only for demonstration purposes and does not \nreflect the opinions or beliefs of Microsoft. Microsoft makes no representations or \nwarranties of any kind, express or implied, about the completeness, accuracy, reliability, \nsuitability or availability with respect to the information contained in this document. \nAll rights reserved to Microsoft \n Contoso Electronics Employee Handbook \nLast Updated: 2023 -03-05 \n \nContoso Electronics is a leader in the aerospace industry, providing advanced electronic \ncomponents for both commercial and military aircraft. We specialize in creating cutting -\nedge systems that are both reliable and efficient. Our mission is to provide the highest \nquality aircraft components to our customers, while maintaining a commitment to safety \nand excellence. We are proud to have built a strong reputation in the aerospace industry \nand strive to continually improve our ", " We are proud to have built a strong reputation in the aerospace industry \nand strive to continually improve our products and services. Our experienced team of \nengineers and technicians are dedicated to providing the best products and services to our \ncustomers. With our commitm ent to excellence, we are sure to remain a leader in the \naerospace industry for years to come. \nOur Mission \n \nContoso Electronics is a leader in the aerospace industry, providing advanced electronic \ncomponents for both commercial and military aircraft. We sp ecialize in creating cutting -\nedge systems that are both reliable and efficient. Our mission is to provide the highest \nquality aircraft components to our customers, while maintaining a commitment to safety \nand excellence. We are proud to have built a strong reputation in the aerospace industry \nand strive to continually improve our products and services. Our experienced team of \nengineers and technicians are dedicated to providing the best products and services to our \ncustomers.", diff --git a/tests/test_app.py b/tests/test_app.py index 1d06e00734..208ef830e2 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -169,6 +169,22 @@ async def test_ask_rtr_text(client, snapshot): snapshot.assert_match(json.dumps(result, indent=4), "result.json") +@pytest.mark.asyncio +async def test_ask_rtr_text_agent(agent_client, snapshot): + response = await agent_client.post( + "/ask", + json={ + "messages": [{"content": "What is the capital of France?", "role": "user"}], + "context": { + "overrides": {"retrieval_mode": "text", "use_agentic_retrieval": True}, + }, + }, + ) + assert response.status_code == 200 + result = await response.get_json() + snapshot.assert_match(json.dumps(result, indent=4), "result.json") + + @pytest.mark.asyncio async def test_ask_rtr_text_filter(auth_client, snapshot): response = await auth_client.post( @@ -195,6 +211,33 @@ async def test_ask_rtr_text_filter(auth_client, snapshot): snapshot.assert_match(json.dumps(result, indent=4), "result.json") +@pytest.mark.asyncio +async def test_ask_rtr_text_agent_filter(agent_auth_client, snapshot): + response = await agent_auth_client.post( + "/ask", + headers={"Authorization": "Bearer MockToken"}, + json={ + "messages": [{"content": "What is the capital of France?", "role": "user"}], + "context": { + "overrides": { + "retrieval_mode": "text", + "use_oid_security_filter": True, + "use_groups_security_filter": True, + "exclude_category": "excluded", + "use_agentic_retrieval": True, + }, + }, + }, + ) + assert response.status_code == 200 + assert ( + agent_auth_client.config[app.CONFIG_AGENT_CLIENT].filter + == "category ne 'excluded' and (oids/any(g:search.in(g, 'OID_X')) or groups/any(g:search.in(g, 'GROUP_Y, GROUP_Z')))" + ) + result = await response.get_json() + snapshot.assert_match(json.dumps(result, indent=4), "result.json") + + @pytest.mark.asyncio async def test_ask_rtr_text_filter_public_documents(auth_public_documents_client, snapshot): response = await auth_public_documents_client.post( @@ -464,6 +507,24 @@ async def test_chat_text(client, snapshot): snapshot.assert_match(json.dumps(result, indent=4), "result.json") +@pytest.mark.asyncio +async def test_chat_text_agent(agent_client, snapshot): + response = await agent_client.post( + "/chat", + json={ + "messages": [{"content": "What is the capital of France?", "role": "user"}], + "context": { + "overrides": {"use_agentic_retrieval": True}, + }, + }, + ) + assert response.status_code == 200 + result = await response.get_json() + assert result["context"]["thoughts"][0]["props"]["max_docs_for_reranker"] == 500 + assert result["context"]["thoughts"][0]["props"]["reranker_threshold"] == 0 + snapshot.assert_match(json.dumps(result, indent=4), "result.json") + + @pytest.mark.asyncio async def test_chat_text_filter(auth_client, snapshot): response = await auth_client.post( @@ -490,6 +551,32 @@ async def test_chat_text_filter(auth_client, snapshot): snapshot.assert_match(json.dumps(result, indent=4), "result.json") +@pytest.mark.asyncio +async def test_chat_text_filter_agent(agent_auth_client, snapshot): + response = await agent_auth_client.post( + "/chat", + headers={"Authorization": "Bearer MockToken"}, + json={ + "messages": [{"content": "What is the capital of France?", "role": "user"}], + "context": { + "overrides": { + "use_agentic_retrieval": True, + "use_oid_security_filter": True, + "use_groups_security_filter": True, + "exclude_category": "excluded", + }, + }, + }, + ) + assert response.status_code == 200 + assert ( + agent_auth_client.config[app.CONFIG_AGENT_CLIENT].filter + == "category ne 'excluded' and (oids/any(g:search.in(g, 'OID_X')) or groups/any(g:search.in(g, 'GROUP_Y, GROUP_Z')))" + ) + result = await response.get_json() + snapshot.assert_match(json.dumps(result, indent=4), "result.json") + + @pytest.mark.asyncio async def test_chat_text_filter_public_documents(auth_public_documents_client, snapshot): response = await auth_public_documents_client.post( diff --git a/tests/test_chatapproach.py b/tests/test_chatapproach.py index 9900ae88bc..c9417d7b0e 100644 --- a/tests/test_chatapproach.py +++ b/tests/test_chatapproach.py @@ -2,6 +2,7 @@ import pytest from azure.core.credentials import AzureKeyCredential +from azure.search.documents.agent.aio import KnowledgeAgentRetrievalClient from azure.search.documents.aio import SearchClient from openai.types.chat import ChatCompletion @@ -12,6 +13,7 @@ MOCK_EMBEDDING_DIMENSIONS, MOCK_EMBEDDING_MODEL_NAME, MockAsyncSearchResultsIterator, + mock_retrieval_response, ) @@ -19,10 +21,18 @@ async def mock_search(*args, **kwargs): return MockAsyncSearchResultsIterator(kwargs.get("search_text"), kwargs.get("vector_queries")) +async def mock_retrieval(*args, **kwargs): + return mock_retrieval_response() + + @pytest.fixture def chat_approach(): return ChatReadRetrieveReadApproach( search_client=None, + search_index_name=None, + agent_model=None, + agent_deployment=None, + agent_client=None, auth_helper=None, openai_client=None, chatgpt_model="gpt-4o-mini", @@ -170,6 +180,10 @@ async def test_search_results_filtering_by_scores( chat_approach = ChatReadRetrieveReadApproach( search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), + search_index_name=None, + agent_model=None, + agent_deployment=None, + agent_client=None, auth_helper=None, openai_client=None, chatgpt_model="gpt-4o-mini", @@ -209,6 +223,10 @@ async def test_search_results_filtering_by_scores( async def test_search_results_query_rewriting(monkeypatch): chat_approach = ChatReadRetrieveReadApproach( search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), + search_index_name=None, + agent_model=None, + agent_deployment=None, + agent_client=None, auth_helper=None, openai_client=None, chatgpt_model="gpt-35-turbo", @@ -246,3 +264,39 @@ async def validate_qr_and_mock_search(*args, **kwargs): ) assert len(results) == 1 assert query_rewrites == "generative" + + +@pytest.mark.asyncio +async def test_agent_retrieval_results(monkeypatch): + chat_approach = ChatReadRetrieveReadApproach( + search_client=None, + search_index_name=None, + agent_model=None, + agent_deployment=None, + agent_client=None, + auth_helper=None, + openai_client=None, + chatgpt_model="gpt-35-turbo", + chatgpt_deployment="chat", + embedding_deployment="embeddings", + embedding_model=MOCK_EMBEDDING_MODEL_NAME, + embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, + embedding_field="embedding3", + sourcepage_field="", + content_field="", + query_language="en-us", + query_speller="lexicon", + prompt_manager=PromptyManager(), + ) + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + + _, results = await chat_approach.run_agentic_retrieval(messages=[], agent_client=agent_client, search_index_name="") + + assert len(results) == 1 + assert results[0].id == "Benefit_Options-2.pdf" + assert results[0].content == "There is a whistleblower policy." + assert results[0].sourcepage == "Benefit_Options-2.pdf" + assert results[0].search_agent_query == "whistleblower query"