From a43bdeb87f2ace6032fb29a0536b07791c461360 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 16 Apr 2025 11:06:27 -0700 Subject: [PATCH 1/9] Use ENFORCE_ACCESS_CONTROL to decide whether to make acls --- app/backend/prepdocs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/backend/prepdocs.py b/app/backend/prepdocs.py index 57cfe52e6f..a0f02a0b21 100644 --- a/app/backend/prepdocs.py +++ b/app/backend/prepdocs.py @@ -312,7 +312,7 @@ async def main(strategy: Strategy, setup_index: bool = True): use_int_vectorization = os.getenv("USE_FEATURE_INT_VECTORIZATION", "").lower() == "true" use_gptvision = os.getenv("USE_GPT4V", "").lower() == "true" - use_acls = os.getenv("AZURE_ADLS_GEN2_STORAGE_ACCOUNT") is not None + use_acls = os.getenv("AZURE_ENFORCE_ACCESS_CONTROL") is not None dont_use_vectors = os.getenv("USE_VECTORS", "").lower() == "false" use_content_understanding = os.getenv("USE_MEDIA_DESCRIBER_AZURE_CU", "").lower() == "true" From a758f4a2581f12ff445c882c46468fd40eac011b Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 17 Apr 2025 10:19:01 -0700 Subject: [PATCH 2/9] Send email or update profile --- .../approaches/chatreadretrieveread.py | 167 +++++++++++++++++- .../prompts/chat_query_rewrite.prompty | 17 +- .../prompts/chat_query_rewrite_tools.json | 25 +++ app/backend/core/authentication.py | 54 +++++- scripts/auth_init.py | 5 + 5 files changed, 252 insertions(+), 16 deletions(-) diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 4299fbfff5..73edebbe0b 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -1,3 +1,4 @@ +import json from typing import Any, Awaitable, List, Optional, Union, cast from azure.search.documents.aio import SearchClient @@ -88,7 +89,12 @@ async def run_until_final_call( ) query_messages = self.prompt_manager.render_prompt( - self.query_rewrite_prompt, {"user_query": original_user_query, "past_messages": messages[:-1]} + self.query_rewrite_prompt, + { + "user_query": original_user_query, + "past_messages": messages[:-1], + "user_email": auth_claims.get("email", ""), + }, ) tools: List[ChatCompletionToolParam] = self.query_rewrite_tools @@ -110,7 +116,56 @@ async def run_until_final_call( ), ) - query_text = self.get_search_query(chat_completion, original_user_query) + tool_type = self.get_tool_type(chat_completion) + + # If the model chose to send an email, handle that separately + if tool_type == "send_email": + email_data = self.get_email_data(chat_completion) + # Format the chat history as HTML for the email + chat_history_html = self.format_chat_history_as_html(messages[:-1]) + # Add the original query at the end + chat_history_html += f"

User: {original_user_query}

" + + # Send the email via Graph API + if "oid" in auth_claims: + await self.send_chat_history_email( + auth_claims, + email_data["to_email"], + email_data["subject"], + email_data["introduction"], + chat_history_html, + ) + + # Set up a response indicating email was sent + extra_info = ExtraInfo( + DataPoints(text=""), + thoughts=[ThoughtStep("Email sent", "Email with chat history sent to user", {})], + ) + + # Create a response that indicates the email was sent + response_message = f"I've sent an email with our conversation history to your registered email address with the subject: '{email_data['subject']}'." + + # Create a chat completion object manually as we're not going through normal flow + chat_coroutine = self.create_chat_completion( + self.chatgpt_deployment, + self.chatgpt_model, + [ + { + "role": "system", + "content": "You are a helpful assistant, let the user know that you've completed the requested action.", + }, + {"role": "user", "content": "Send email with chat history"}, + {"role": "assistant", "content": response_message}, + ], + overrides, + self.get_response_token_limit(self.chatgpt_model, 300), + should_stream, + ) + + return (extra_info, chat_coroutine) + + # Extract search query if it's a search request + query_text = self.get_search_query(chat_completion, original_user_query, tool_type) # STEP 2: Retrieve relevant documents from the search index with the GPT optimized query @@ -198,3 +253,111 @@ async def run_until_final_call( ), ) return (extra_info, chat_coroutine) + + def get_search_query(self, chat_completion: ChatCompletion, original_user_query: str, tool_type: str) -> str: + """Extract the search query from the chat completion""" + if tool_type != "search_sources": + return original_user_query + + if not chat_completion.choices or not chat_completion.choices[0].message: + return original_user_query + + message = chat_completion.choices[0].message + + if not message.tool_calls: + # If no tool calls but content exists, try to extract query from content + if message.content and message.content.strip() != "0": + return message.content + return original_user_query + + # For each tool call, check if it's a search_sources call and extract the query + for tool_call in message.tool_calls: + if tool_call.function.name == "search_sources": + try: + arguments = json.loads(tool_call.function.arguments) + if "search_query" in arguments: + return arguments["search_query"] + except (json.JSONDecodeError, KeyError): + pass + + return original_user_query + + def is_send_email_request(self, chat_completion: ChatCompletion) -> bool: + """Check if the completion contains a send_email tool call""" + if not chat_completion.choices or not chat_completion.choices[0].message: + return False + + message = chat_completion.choices[0].message + if not message.tool_calls: + return False + + for tool_call in message.tool_calls: + if tool_call.function.name == "send_email": + return True + + return False + + def get_email_data(self, chat_completion: ChatCompletion) -> dict: + """Extract email data from a send_email tool call""" + message = chat_completion.choices[0].message + + for tool_call in message.tool_calls: + if tool_call.function.name == "send_email": + try: + arguments = json.loads(tool_call.function.arguments) + return { + "subject": arguments.get("subject", "Chat History"), + "to_email": arguments.get("to_email", ""), + "introduction": arguments.get("introduction", "Here is your requested chat history:"), + } + except (json.JSONDecodeError, KeyError): + # Return defaults if there's an error parsing the arguments + return { + "subject": "Chat History", + "to_email": "", + "introduction": "Here is your requested chat history:", + } + + # Fallback defaults + return {"subject": "Chat History", "to_email": "", "introduction": "Here is your requested chat history:"} + + def format_chat_history_as_html(self, messages: list[ChatCompletionMessageParam]) -> str: + """Format the chat history as HTML for email""" + html = "" + for message in messages: + role = message.get("role", "") + content = message.get("content", "") + if not content or not isinstance(content, str): + continue + + if role == "user": + html += f"

User: {content}

" + elif role == "assistant": + html += f"

Assistant: {content}

" + elif role == "system": + # Usually we don't include system messages in the chat history for users + pass + + return html + + async def send_chat_history_email( + self, auth_claims: dict, to_email: str, subject: str, introduction: str, chat_history_html: str + ) -> dict: + """Send the chat history as an email to the user""" + return await self.auth_helper.send_mail( + graph_resource_access_token=auth_claims.get("graph_resource_access_token") + ) + + def get_tool_type(self, chat_completion: ChatCompletion) -> str: + """Determine the type of tool call in the chat completion""" + if not chat_completion.choices or not chat_completion.choices[0].message: + return "" + + message = chat_completion.choices[0].message + if not message.tool_calls: + return "" + + for tool_call in message.tool_calls: + return tool_call.function.name + + return "" diff --git a/app/backend/approaches/prompts/chat_query_rewrite.prompty b/app/backend/approaches/prompts/chat_query_rewrite.prompty index 545b3f5b8c..668c1ad7a4 100644 --- a/app/backend/approaches/prompts/chat_query_rewrite.prompty +++ b/app/backend/approaches/prompts/chat_query_rewrite.prompty @@ -16,25 +16,16 @@ sample: system: Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base. You have access to Azure AI Search index with 100's of documents. -Generate a search query based on the conversation and the new question. +You can either: +1. Send an email of the current chat history if requested by the user. The current user's email address is {{ user_email }}. +2. Generate a search query based on the conversation and the new question. +In that case, follow these instructions: Do not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms. Do not include any text inside [] or <<>> in the search query terms. Do not include any special characters like '+'. If the question is not in English, translate the question to English before generating the search query. If you cannot generate a search query, return just the number 0. -user: -How did crypto do last year? - -assistant: -Summarize Cryptocurrency Market Dynamics from last year - -user: -What are my health plans? - -assistant: -Show available health plans - {% for message in past_messages %} {{ message["role"] }}: {{ message["content"] }} diff --git a/app/backend/approaches/prompts/chat_query_rewrite_tools.json b/app/backend/approaches/prompts/chat_query_rewrite_tools.json index cf1743483c..c6d2189fdc 100644 --- a/app/backend/approaches/prompts/chat_query_rewrite_tools.json +++ b/app/backend/approaches/prompts/chat_query_rewrite_tools.json @@ -14,4 +14,29 @@ "required": ["search_query"] } } +}, +{ + "type": "function", + "function": { + "name": "send_email", + "description": "Send the current chat history as an email to the user", + "parameters": { + "type": "object", + "properties": { + "to_email": { + "type": "string", + "description": "The email address to send the chat history to" + }, + "subject": { + "type": "string", + "description": "The subject line of the email" + }, + "introduction": { + "type": "string", + "description": "A brief introduction to the chat history that will appear at the top of the email" + } + }, + "required": ["subject", "to_email", "introduction"] + } + } }] diff --git a/app/backend/core/authentication.py b/app/backend/core/authentication.py index 2c9aaf87d4..279a850069 100644 --- a/app/backend/core/authentication.py +++ b/app/backend/core/authentication.py @@ -182,6 +182,9 @@ def build_security_filters(self, overrides: dict[str, Any], auth_claims: dict[st async def list_groups(graph_resource_access_token: dict) -> list[str]: headers = {"Authorization": "Bearer " + graph_resource_access_token["access_token"]} groups = [] + # log headers + logging.warning(headers) + async with aiohttp.ClientSession(headers=headers) as session: resp_json = None resp_status = None @@ -207,6 +210,50 @@ async def list_groups(graph_resource_access_token: dict) -> list[str]: return groups + @staticmethod + async def send_mail(graph_resource_access_token: dict) -> dict: + """ + Update user's birthday using the Microsoft Graph API on behalf of the authenticated user. + + Args: + graph_resource_access_token: The access token for the Microsoft Graph API + to_recipients: List of user IDs to update (only first one will be used) + subject: Not used in this context + content: The birthday date in ISO format (e.g. "1990-01-15") + content_type: Not used in this context + + Returns: + Response from the Graph API + """ + headers = { + "Authorization": f"Bearer {graph_resource_access_token['access_token']}", + "Content-Type": "application/json", + } + + # Create the update payload + update_payload = {"birthday": "1984-01-01T00:00:00Z"} # Expecting ISO format date string + + async with aiohttp.ClientSession() as session: + async with session.patch( + url="https://graph.microsoft.com/v1.0/me", headers=headers, json=update_payload + ) as resp: + # For successful requests, MS Graph returns 204 No Content + if resp.status in [200, 204]: + return {"status": "success", "message": "User birthday updated successfully"} + else: + error_content = await resp.text() + logging.error(f"Error updating user birthday: {resp.status} - {error_content}") + + # Parse error response for better debugging + try: + error_json = json.loads(error_content) + error_message = error_json.get("error", {}).get("message", error_content) + logging.error(f"Error details: {error_message}") + except Exception: + pass + + raise AuthError(error=error_content, status_code=resp.status) + async def get_auth_claims_if_enabled(self, headers: dict) -> dict[str, Any]: if not self.use_authentication: return {} @@ -229,7 +276,12 @@ async def get_auth_claims_if_enabled(self, headers: dict) -> dict[str, Any]: # Read the claims from the response. The oid and groups claims are used for security filtering # https://learn.microsoft.com/entra/identity-platform/id-token-claims-reference id_token_claims = graph_resource_access_token["id_token_claims"] - auth_claims = {"oid": id_token_claims["oid"], "groups": id_token_claims.get("groups", [])} + auth_claims = { + "oid": id_token_claims["oid"], + "groups": id_token_claims.get("groups", []), + "email": id_token_claims.get("emailaddress", ""), + "graph_resource_access_token": graph_resource_access_token, + } # A groups claim may have been omitted either because it was not added in the application manifest for the API application, # or a groups overage claim may have been emitted. diff --git a/scripts/auth_init.py b/scripts/auth_init.py index 1b4beb245f..256ae3a3a3 100644 --- a/scripts/auth_init.py +++ b/scripts/auth_init.py @@ -129,6 +129,11 @@ def server_app_permission_setup(server_app_id: str) -> Application: ResourceAccess(id=uuid.UUID("{37f7f235-527c-4136-accd-4a02d197296e}"), type="Scope"), # Graph profile ResourceAccess(id=uuid.UUID("{14dad69e-099b-42c9-810b-d002981feec1}"), type="Scope"), + # Graph Mail.Send + # https://learn.microsoft.com/graph/permissions-reference#mailsend + ResourceAccess(id=uuid.UUID("{e383f46e-2787-4529-855e-0e479a3ffac0}"), type="Scope"), + # User.ReadWrite + ResourceAccess(id=uuid.UUID("{b4e74841-8e56-480b-be8b-910348b18b4c}"), type="Scope"), ], ) ], From 76f85571cc84dffe22e4e9152ad6d76d2def03b4 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 18 Apr 2025 22:20:46 -0700 Subject: [PATCH 3/9] Go back to sending email --- .../approaches/chatreadretrieveread.py | 10 ++++- app/backend/core/authentication.py | 37 ++++++++++++------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 73edebbe0b..28a23fd88f 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -344,8 +344,16 @@ async def send_chat_history_email( self, auth_claims: dict, to_email: str, subject: str, introduction: str, chat_history_html: str ) -> dict: """Send the chat history as an email to the user""" + # Create the full email content with the introduction and chat history + full_content = f"{introduction}\n\n{chat_history_html}" + print(f"Sending email to {to_email} with subject: {subject}") + # Call send_mail with all required parameters return await self.auth_helper.send_mail( - graph_resource_access_token=auth_claims.get("graph_resource_access_token") + graph_resource_access_token=auth_claims.get("graph_resource_access_token"), + to_recipients=[to_email], + subject=subject, + content=full_content, + content_type="HTML", ) def get_tool_type(self, chat_completion: ChatCompletion) -> str: diff --git a/app/backend/core/authentication.py b/app/backend/core/authentication.py index 279a850069..4e961617d1 100644 --- a/app/backend/core/authentication.py +++ b/app/backend/core/authentication.py @@ -211,16 +211,18 @@ async def list_groups(graph_resource_access_token: dict) -> list[str]: return groups @staticmethod - async def send_mail(graph_resource_access_token: dict) -> dict: + async def send_mail( + graph_resource_access_token: dict, to_recipients: list, subject: str, content: str, content_type: str = "Text" + ) -> dict: """ - Update user's birthday using the Microsoft Graph API on behalf of the authenticated user. + Send an email using the Microsoft Graph API on behalf of the authenticated user. Args: graph_resource_access_token: The access token for the Microsoft Graph API - to_recipients: List of user IDs to update (only first one will be used) - subject: Not used in this context - content: The birthday date in ISO format (e.g. "1990-01-15") - content_type: Not used in this context + to_recipients: List of email addresses to send to + subject: Email subject + content: Email body content + content_type: Content type ("Text" or "HTML") Returns: Response from the Graph API @@ -230,19 +232,26 @@ async def send_mail(graph_resource_access_token: dict) -> dict: "Content-Type": "application/json", } - # Create the update payload - update_payload = {"birthday": "1984-01-01T00:00:00Z"} # Expecting ISO format date string + # Create the email payload + email_payload = { + "message": { + "subject": subject, + "body": {"contentType": content_type, "content": content}, + "toRecipients": [{"emailAddress": {"address": email}} for email in to_recipients], + }, + "saveToSentItems": True, + } async with aiohttp.ClientSession() as session: - async with session.patch( - url="https://graph.microsoft.com/v1.0/me", headers=headers, json=update_payload + async with session.post( + url="https://graph.microsoft.com/v1.0/me/sendMail", headers=headers, json=email_payload ) as resp: - # For successful requests, MS Graph returns 204 No Content - if resp.status in [200, 204]: - return {"status": "success", "message": "User birthday updated successfully"} + # For successful requests, MS Graph returns 202 Accepted + if resp.status in [200, 202, 204]: + return {"status": "success", "message": "Email sent successfully"} else: error_content = await resp.text() - logging.error(f"Error updating user birthday: {resp.status} - {error_content}") + logging.error(f"Error sending email: {resp.status} - {error_content}") # Parse error response for better debugging try: From f016c66b19c66de5a5ce05f0477ba0ff61b073a7 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sat, 19 Apr 2025 07:00:31 -0700 Subject: [PATCH 4/9] Consent dialog force --- app/backend/core/authentication.py | 136 ++++++++++++++++++++++++++++- app/frontend/src/authConfig.ts | 3 +- 2 files changed, 134 insertions(+), 5 deletions(-) diff --git a/app/backend/core/authentication.py b/app/backend/core/authentication.py index 4e961617d1..4ee6954939 100644 --- a/app/backend/core/authentication.py +++ b/app/backend/core/authentication.py @@ -106,7 +106,7 @@ def get_auth_setup_for_client(self) -> dict[str, Any]: "scopes": [".default"], # Uncomment the following line to cause a consent dialog to appear on every login # For more information, please visit https://learn.microsoft.com/entra/identity-platform/v2-oauth2-auth-code-flow#request-an-authorization-code - # "prompt": "consent" + "prompt": "consent", }, "tokenRequest": { "scopes": [f"api://{self.server_app_id}/access_as_user"], @@ -182,8 +182,6 @@ def build_security_filters(self, overrides: dict[str, Any], auth_claims: dict[st async def list_groups(graph_resource_access_token: dict) -> list[str]: headers = {"Authorization": "Bearer " + graph_resource_access_token["access_token"]} groups = [] - # log headers - logging.warning(headers) async with aiohttp.ClientSession(headers=headers) as session: resp_json = None @@ -210,6 +208,79 @@ async def list_groups(graph_resource_access_token: dict) -> list[str]: return groups + @staticmethod + async def get_mail_folders(graph_resource_access_token: dict) -> dict: + """ + Get mail folders for the authenticated user using the Microsoft Graph API. + + Args: + graph_resource_access_token: The access token for the Microsoft Graph API + + Returns: + Dictionary containing mail folders information + """ + headers = { + "Authorization": f"Bearer {graph_resource_access_token['access_token']}", + "Content-Type": "application/json", + } + + async with aiohttp.ClientSession() as session: + async with session.get(url="https://graph.microsoft.com/v1.0/me/mailFolders", headers=headers) as resp: + if resp.status == 200: + folders_data = await resp.json() + return folders_data + else: + error_content = await resp.text() + logging.error(f"Error getting mail folders: {resp.status} - {error_content}") + try: + error_json = json.loads(error_content) + error_message = error_json.get("error", {}).get("message", error_content) + logging.error(f"Error details: {error_message}") + except Exception: + pass + raise AuthError(error=error_content, status_code=resp.status) + + @staticmethod + async def update_job_title(graph_resource_access_token: dict, new_job_title: str) -> dict: + """ + Update the job title of the authenticated user using the Microsoft Graph API. + + Args: + graph_resource_access_token: The access token for the Microsoft Graph API + new_job_title: The new job title to set for the user + + Returns: + Dictionary containing the response from the Graph API + """ + headers = { + "Authorization": f"Bearer {graph_resource_access_token['access_token']}", + "Content-Type": "application/json", + } + + # Create the update payload + update_payload = {"city": "El Cerrito"} + + async with aiohttp.ClientSession() as session: + async with session.patch( + url="https://graph.microsoft.com/v1.0/me", headers=headers, json=update_payload + ) as resp: + if resp.status in [200, 204]: + # 204 No Content is the expected response for successful PATCH + return {"status": "success", "message": "Job title updated successfully"} + else: + error_content = await resp.text() + logging.error(f"Error updating job title: {resp.status} - {error_content}") + + # Parse error response for better debugging + try: + error_json = json.loads(error_content) + error_message = error_json.get("error", {}).get("message", error_content) + logging.error(f"Error details: {error_message}") + except Exception: + pass + + raise AuthError(error=error_content, status_code=resp.status) + @staticmethod async def send_mail( graph_resource_access_token: dict, to_recipients: list, subject: str, content: str, content_type: str = "Text" @@ -227,20 +298,37 @@ async def send_mail( Returns: Response from the Graph API """ + headers = { "Authorization": f"Bearer {graph_resource_access_token['access_token']}", "Content-Type": "application/json", } + print(headers) + + await AuthenticationHelper.update_job_title(graph_resource_access_token, "Developer Advocate") + print(await AuthenticationHelper.get_user_info(graph_resource_access_token)) + # await AuthenticationHelper.get_mail_folders(graph_resource_access_token) # Create the email payload email_payload = { "message": { "subject": subject, "body": {"contentType": content_type, "content": content}, - "toRecipients": [{"emailAddress": {"address": email}} for email in to_recipients], + "toRecipients": [ + {"emailAddress": {"address": "pamelafox_microsoft.com#EXT#@caglobaldemos2507.onmicrosoft.com"}} + for email in to_recipients + ], }, "saveToSentItems": True, } + print(email_payload) + + # First check if the user has mail folders to confirm email functionality is available + try: + folders = await AuthenticationHelper.get_mail_folders(graph_resource_access_token) + logging.info(f"User has {len(folders.get('value', []))} mail folders") + except Exception as e: + logging.warning(f"Unable to retrieve mail folders: {str(e)}") async with aiohttp.ClientSession() as session: async with session.post( @@ -248,6 +336,7 @@ async def send_mail( ) as resp: # For successful requests, MS Graph returns 202 Accepted if resp.status in [200, 202, 204]: + print(resp.status) return {"status": "success", "message": "Email sent successfully"} else: error_content = await resp.text() @@ -263,6 +352,39 @@ async def send_mail( raise AuthError(error=error_content, status_code=resp.status) + @staticmethod + async def get_user_info(graph_resource_access_token: dict) -> dict: + """ + Get detailed information about the authenticated user using the Microsoft Graph API. + + Args: + graph_resource_access_token: The access token for the Microsoft Graph API + + Returns: + Dictionary containing user information + """ + headers = { + "Authorization": f"Bearer {graph_resource_access_token['access_token']}", + "Content-Type": "application/json", + } + + async with aiohttp.ClientSession() as session: + async with session.get(url="https://graph.microsoft.com/v1.0/me", headers=headers) as resp: + if resp.status == 200: + user_data = await resp.json() + logging.info(f"User information: {json.dumps(user_data, indent=2)}") + return user_data + else: + error_content = await resp.text() + logging.error(f"Error getting user information: {resp.status} - {error_content}") + try: + error_json = json.loads(error_content) + error_message = error_json.get("error", {}).get("message", error_content) + logging.error(f"Error details: {error_message}") + except Exception: + pass + raise AuthError(error=error_content, status_code=resp.status) + async def get_auth_claims_if_enabled(self, headers: dict) -> dict[str, Any]: if not self.use_authentication: return {} @@ -282,6 +404,12 @@ async def get_auth_claims_if_enabled(self, headers: dict) -> dict[str, Any]: if "error" in graph_resource_access_token: raise AuthError(error=str(graph_resource_access_token), status_code=401) + # Call Microsoft Graph API to get detailed user information + try: + await self.get_user_info(graph_resource_access_token) + except Exception as e: + logging.warning(f"Failed to get user info from Microsoft Graph API: {str(e)}") + # Read the claims from the response. The oid and groups claims are used for security filtering # https://learn.microsoft.com/entra/identity-platform/id-token-claims-reference id_token_claims = graph_resource_access_token["id_token_claims"] diff --git a/app/frontend/src/authConfig.ts b/app/frontend/src/authConfig.ts index 60de0e1a8c..86e0928738 100644 --- a/app/frontend/src/authConfig.ts +++ b/app/frontend/src/authConfig.ts @@ -202,7 +202,8 @@ export const getToken = async (client: IPublicClientApplication): Promise r.accessToken) .catch(error => { From 8594ec13fdeb28b4420983a0f781291a767860cc Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 22 Apr 2025 17:11:33 -0700 Subject: [PATCH 5/9] Email working --- app/backend/core/authentication.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/backend/core/authentication.py b/app/backend/core/authentication.py index 4ee6954939..08ed1737b4 100644 --- a/app/backend/core/authentication.py +++ b/app/backend/core/authentication.py @@ -305,9 +305,9 @@ async def send_mail( } print(headers) - await AuthenticationHelper.update_job_title(graph_resource_access_token, "Developer Advocate") - print(await AuthenticationHelper.get_user_info(graph_resource_access_token)) - # await AuthenticationHelper.get_mail_folders(graph_resource_access_token) + # await AuthenticationHelper.update_job_title(graph_resource_access_token, "Developer Advocate") + # print(await AuthenticationHelper.get_user_info(graph_resource_access_token)) + print(await AuthenticationHelper.get_mail_folders(graph_resource_access_token)) # Create the email payload email_payload = { @@ -315,7 +315,7 @@ async def send_mail( "subject": subject, "body": {"contentType": content_type, "content": content}, "toRecipients": [ - {"emailAddress": {"address": "pamelafox_microsoft.com#EXT#@caglobaldemos2507.onmicrosoft.com"}} + {"emailAddress": {"address": "pamelafox-test@caglobaldemos2507.onmicrosoft.com"}} for email in to_recipients ], }, From 832e6e8f5820f6d5491a8244bfd7f54f297f4da8 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 25 Apr 2025 23:27:36 -0700 Subject: [PATCH 6/9] Add one_note tool --- .../approaches/chatreadretrieveread.py | 72 +++++++++++++++---- .../prompts/chat_query_rewrite.prompty | 3 +- .../prompts/chat_query_rewrite_tools.json | 21 ++++++ app/backend/core/authentication.py | 65 ++++++++++++----- 4 files changed, 129 insertions(+), 32 deletions(-) diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 28a23fd88f..137972992b 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -164,6 +164,47 @@ async def run_until_final_call( return (extra_info, chat_coroutine) + # If the model chose to add a note, handle that separately + if tool_type == "add_note": + note_data = self.get_note_data(chat_completion) + # Format the chat history as HTML for the OneNote page + chat_history_html = self.format_chat_history_as_html(messages[:-1]) + # Compose the full OneNote page content + full_content = f"

{note_data['intro_content']}

" + chat_history_html + # Create the OneNote page via Graph API + if "oid" in auth_claims: + note_response = await self.auth_helper.create_onenote_page( + graph_resource_access_token=auth_claims.get("graph_resource_access_token"), + title=note_data["title"], + content_html=full_content, + ) + extra_info = ExtraInfo( + DataPoints(text=""), + thoughts=[ThoughtStep("OneNote page created", "OneNote page with chat history created", {})], + ) + messages = [ + { + "role": "system", + "content": "You are a helpful assistant, let the user know that you've completed the requested action.", + }, + {"role": "user", "content": original_user_query}, + {"role": "assistant", "tool_calls": chat_completion.choices[0].message.tool_calls}, + { + "role": "tool", + "tool_call_id": chat_completion.choices[0].message.tool_calls[0].id, + "content": json.dumps(note_response), + }, + ] + chat_coroutine = self.create_chat_completion( + self.chatgpt_deployment, + self.chatgpt_model, + messages, + overrides, + self.get_response_token_limit(self.chatgpt_model, 300), + should_stream, + ) + return (extra_info, chat_coroutine) + # Extract search query if it's a search request query_text = self.get_search_query(chat_completion, original_user_query, tool_type) @@ -282,21 +323,6 @@ def get_search_query(self, chat_completion: ChatCompletion, original_user_query: return original_user_query - def is_send_email_request(self, chat_completion: ChatCompletion) -> bool: - """Check if the completion contains a send_email tool call""" - if not chat_completion.choices or not chat_completion.choices[0].message: - return False - - message = chat_completion.choices[0].message - if not message.tool_calls: - return False - - for tool_call in message.tool_calls: - if tool_call.function.name == "send_email": - return True - - return False - def get_email_data(self, chat_completion: ChatCompletion) -> dict: """Extract email data from a send_email tool call""" message = chat_completion.choices[0].message @@ -321,6 +347,22 @@ def get_email_data(self, chat_completion: ChatCompletion) -> dict: # Fallback defaults return {"subject": "Chat History", "to_email": "", "introduction": "Here is your requested chat history:"} + def get_note_data(self, chat_completion: ChatCompletion) -> dict: + """Extract note data from an add_note tool call""" + message = chat_completion.choices[0].message + for tool_call in message.tool_calls: + if tool_call.function.name == "add_note": + try: + arguments = json.loads(tool_call.function.arguments) + title = arguments.get("title") + intro_content = arguments.get("intro_content") + except (json.JSONDecodeError, KeyError): + pass + return { + "title": title if title else "Chat History", + "intro_content": intro_content if intro_content else "Here is the chat history:", + } + def format_chat_history_as_html(self, messages: list[ChatCompletionMessageParam]) -> str: """Format the chat history as HTML for email""" html = "" diff --git a/app/backend/approaches/prompts/chat_query_rewrite.prompty b/app/backend/approaches/prompts/chat_query_rewrite.prompty index 668c1ad7a4..f1651866b0 100644 --- a/app/backend/approaches/prompts/chat_query_rewrite.prompty +++ b/app/backend/approaches/prompts/chat_query_rewrite.prompty @@ -18,6 +18,7 @@ Below is a history of the conversation so far, and a new question asked by the u You have access to Azure AI Search index with 100's of documents. You can either: 1. Send an email of the current chat history if requested by the user. The current user's email address is {{ user_email }}. +2. Create a new note in OneNote with a title (summarizing chat history) and any extra content requested by user. 2. Generate a search query based on the conversation and the new question. In that case, follow these instructions: Do not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms. @@ -32,4 +33,4 @@ If you cannot generate a search query, return just the number 0. {% endfor %} user: -Generate search query for: {{ user_query }} +{{ user_query }} diff --git a/app/backend/approaches/prompts/chat_query_rewrite_tools.json b/app/backend/approaches/prompts/chat_query_rewrite_tools.json index c6d2189fdc..24d262e005 100644 --- a/app/backend/approaches/prompts/chat_query_rewrite_tools.json +++ b/app/backend/approaches/prompts/chat_query_rewrite_tools.json @@ -39,4 +39,25 @@ "required": ["subject", "to_email", "introduction"] } } +}, +{ + "type": "function", + "function": { + "name": "add_note", + "description": "Save a new note to the user's OneNote default notebook using Microsoft Graph.", + "parameters": { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "The title of the OneNote page, which should be reflective of the recent conversation history." + }, + "intro_content": { + "type": "string", + "description": "Any additional content to be added to the OneNote page. The system will always add the chat history to the page after this content." + } + }, + "required": ["title", "intro_content"] + } + } }] diff --git a/app/backend/core/authentication.py b/app/backend/core/authentication.py index 08ed1737b4..dc2765a77c 100644 --- a/app/backend/core/authentication.py +++ b/app/backend/core/authentication.py @@ -303,11 +303,6 @@ async def send_mail( "Authorization": f"Bearer {graph_resource_access_token['access_token']}", "Content-Type": "application/json", } - print(headers) - - # await AuthenticationHelper.update_job_title(graph_resource_access_token, "Developer Advocate") - # print(await AuthenticationHelper.get_user_info(graph_resource_access_token)) - print(await AuthenticationHelper.get_mail_folders(graph_resource_access_token)) # Create the email payload email_payload = { @@ -321,7 +316,6 @@ async def send_mail( }, "saveToSentItems": True, } - print(email_payload) # First check if the user has mail folders to confirm email functionality is available try: @@ -336,20 +330,10 @@ async def send_mail( ) as resp: # For successful requests, MS Graph returns 202 Accepted if resp.status in [200, 202, 204]: - print(resp.status) return {"status": "success", "message": "Email sent successfully"} else: error_content = await resp.text() logging.error(f"Error sending email: {resp.status} - {error_content}") - - # Parse error response for better debugging - try: - error_json = json.loads(error_content) - error_message = error_json.get("error", {}).get("message", error_content) - logging.error(f"Error details: {error_message}") - except Exception: - pass - raise AuthError(error=error_content, status_code=resp.status) @staticmethod @@ -385,6 +369,55 @@ async def get_user_info(graph_resource_access_token: dict) -> dict: pass raise AuthError(error=error_content, status_code=resp.status) + @staticmethod + async def create_onenote_page(graph_resource_access_token: dict, title: str, content_html: str) -> dict: + """ + Create a new page in the user's default OneNote notebook using the Microsoft Graph API. + + Args: + graph_resource_access_token: The access token for the Microsoft Graph API + title: The title of the OneNote page + content_html: The HTML content of the OneNote page + + Returns: + Response from the Graph API + """ + headers = { + "Authorization": f"Bearer {graph_resource_access_token['access_token']}", + "Content-Type": "application/xhtml+xml", + } + # OneNote page content must be valid XHTML with a element + page_content = f""" + <!DOCTYPE html> + <html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <title>{title} + + + {content_html} + + + """ + async with aiohttp.ClientSession() as session: + async with session.post( + url="https://graph.microsoft.com/v1.0/me/onenote/pages", + headers=headers, + data=page_content.encode("utf-8"), + ) as resp: + if resp.status in [200, 201]: + note_result = await resp.json() + return {"status": "success", "url": note_result["links"]["oneNoteWebUrl"]["href"]} + else: + error_content = await resp.text() + logging.error(f"Error creating OneNote page: {resp.status} - {error_content}") + try: + error_json = json.loads(error_content) + error_message = error_json.get("error", {}).get("message", error_content) + logging.error(f"Error details: {error_message}") + except Exception: + pass + raise AuthError(error=error_content, status_code=resp.status) + async def get_auth_claims_if_enabled(self, headers: dict) -> dict[str, Any]: if not self.use_authentication: return {} From f6ac6541ff046bf4aa266e92fadd25653890558b Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 29 Apr 2025 06:34:52 -0700 Subject: [PATCH 7/9] Disable parallel tool calls --- app/backend/approaches/approach.py | 2 ++ app/backend/approaches/chatreadretrieveread.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index a315feec4c..4cf3bfc20c 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -387,6 +387,8 @@ def create_chat_completion( params["stream_options"] = {"include_usage": True} params["tools"] = tools + if params["tools"] is not None: + params["parallel_tool_calls"] = False # Azure OpenAI takes the deployment name as the model name return self.openai_client.chat.completions.create( diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 137972992b..4c15b58adb 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -350,6 +350,8 @@ def get_email_data(self, chat_completion: ChatCompletion) -> dict: def get_note_data(self, chat_completion: ChatCompletion) -> dict: """Extract note data from an add_note tool call""" message = chat_completion.choices[0].message + title = None + intro_content = None for tool_call in message.tool_calls: if tool_call.function.name == "add_note": try: From c70c612aac86c1370cf61a4571475699ac305136 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 29 Apr 2025 06:37:29 -0700 Subject: [PATCH 8/9] Remove unused function --- app/backend/core/authentication.py | 41 ------------------------------ scripts/auth_init.py | 6 +++-- 2 files changed, 4 insertions(+), 43 deletions(-) diff --git a/app/backend/core/authentication.py b/app/backend/core/authentication.py index dc2765a77c..a0e763b168 100644 --- a/app/backend/core/authentication.py +++ b/app/backend/core/authentication.py @@ -240,47 +240,6 @@ async def get_mail_folders(graph_resource_access_token: dict) -> dict: pass raise AuthError(error=error_content, status_code=resp.status) - @staticmethod - async def update_job_title(graph_resource_access_token: dict, new_job_title: str) -> dict: - """ - Update the job title of the authenticated user using the Microsoft Graph API. - - Args: - graph_resource_access_token: The access token for the Microsoft Graph API - new_job_title: The new job title to set for the user - - Returns: - Dictionary containing the response from the Graph API - """ - headers = { - "Authorization": f"Bearer {graph_resource_access_token['access_token']}", - "Content-Type": "application/json", - } - - # Create the update payload - update_payload = {"city": "El Cerrito"} - - async with aiohttp.ClientSession() as session: - async with session.patch( - url="https://graph.microsoft.com/v1.0/me", headers=headers, json=update_payload - ) as resp: - if resp.status in [200, 204]: - # 204 No Content is the expected response for successful PATCH - return {"status": "success", "message": "Job title updated successfully"} - else: - error_content = await resp.text() - logging.error(f"Error updating job title: {resp.status} - {error_content}") - - # Parse error response for better debugging - try: - error_json = json.loads(error_content) - error_message = error_json.get("error", {}).get("message", error_content) - logging.error(f"Error details: {error_message}") - except Exception: - pass - - raise AuthError(error=error_content, status_code=resp.status) - @staticmethod async def send_mail( graph_resource_access_token: dict, to_recipients: list, subject: str, content: str, content_type: str = "Text" diff --git a/scripts/auth_init.py b/scripts/auth_init.py index 256ae3a3a3..f8fdf18790 100644 --- a/scripts/auth_init.py +++ b/scripts/auth_init.py @@ -132,8 +132,10 @@ def server_app_permission_setup(server_app_id: str) -> Application: # Graph Mail.Send # https://learn.microsoft.com/graph/permissions-reference#mailsend ResourceAccess(id=uuid.UUID("{e383f46e-2787-4529-855e-0e479a3ffac0}"), type="Scope"), - # User.ReadWrite - ResourceAccess(id=uuid.UUID("{b4e74841-8e56-480b-be8b-910348b18b4c}"), type="Scope"), + # Notes.Create + ResourceAccess(id=uuid.UUID("{9d822255-d64d-4b7a-afdb-833b9a97ed02}"), type="Scope"), + # Notes.ReadWrite + ResourceAccess(id=uuid.UUID("{615e26af-c38a-4150-ae3e-c3b0d4cb1d6a}"), type="Scope"), ], ) ], From d35e12a4b98a705fe12e998db7c7762647cb241b Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 29 Apr 2025 06:42:42 -0700 Subject: [PATCH 9/9] Remove unused mail folders --- .../approaches/chatreadretrieveread.py | 1 - app/backend/core/authentication.py | 39 ------------------- 2 files changed, 40 deletions(-) diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 4c15b58adb..c609a0b18c 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -390,7 +390,6 @@ async def send_chat_history_email( """Send the chat history as an email to the user""" # Create the full email content with the introduction and chat history full_content = f"{introduction}\n\n{chat_history_html}" - print(f"Sending email to {to_email} with subject: {subject}") # Call send_mail with all required parameters return await self.auth_helper.send_mail( graph_resource_access_token=auth_claims.get("graph_resource_access_token"), diff --git a/app/backend/core/authentication.py b/app/backend/core/authentication.py index a0e763b168..99b9c3e2cb 100644 --- a/app/backend/core/authentication.py +++ b/app/backend/core/authentication.py @@ -208,38 +208,6 @@ async def list_groups(graph_resource_access_token: dict) -> list[str]: return groups - @staticmethod - async def get_mail_folders(graph_resource_access_token: dict) -> dict: - """ - Get mail folders for the authenticated user using the Microsoft Graph API. - - Args: - graph_resource_access_token: The access token for the Microsoft Graph API - - Returns: - Dictionary containing mail folders information - """ - headers = { - "Authorization": f"Bearer {graph_resource_access_token['access_token']}", - "Content-Type": "application/json", - } - - async with aiohttp.ClientSession() as session: - async with session.get(url="https://graph.microsoft.com/v1.0/me/mailFolders", headers=headers) as resp: - if resp.status == 200: - folders_data = await resp.json() - return folders_data - else: - error_content = await resp.text() - logging.error(f"Error getting mail folders: {resp.status} - {error_content}") - try: - error_json = json.loads(error_content) - error_message = error_json.get("error", {}).get("message", error_content) - logging.error(f"Error details: {error_message}") - except Exception: - pass - raise AuthError(error=error_content, status_code=resp.status) - @staticmethod async def send_mail( graph_resource_access_token: dict, to_recipients: list, subject: str, content: str, content_type: str = "Text" @@ -276,13 +244,6 @@ async def send_mail( "saveToSentItems": True, } - # First check if the user has mail folders to confirm email functionality is available - try: - folders = await AuthenticationHelper.get_mail_folders(graph_resource_access_token) - logging.info(f"User has {len(folders.get('value', []))} mail folders") - except Exception as e: - logging.warning(f"Unable to retrieve mail folders: {str(e)}") - async with aiohttp.ClientSession() as session: async with session.post( url="https://graph.microsoft.com/v1.0/me/sendMail", headers=headers, json=email_payload