Merge pull request #42 from microsoft/adesousa_microsoft/support-citations

data scripts update

Author: andrewldesousa

.gitignore

@@ -7,4 +7,7 @@ __pycache__/
 .ipynb_checkpoints/
 
 data
-static
+static
+
+scripts/config.json
+venv

app.py

@@ -163,7 +163,6 @@ def init_ai_search_client():
         endpoint = app_settings.datasource.endpoint
         key_credential = app_settings.datasource.key
         index_name = app_settings.datasource.index
-
         client = SearchClient(endpoint=endpoint, index_name=index_name, credential=AzureKeyCredential(key_credential))
         return client
     except Exception as e:
@@ -889,10 +888,10 @@ async def generate_section_content():
         logging.exception("Exception in /history/clear_messages")
         return jsonify({"error": str(e)}), 500
 
-@bp.route("/document/<documentId>")
-async def get_document(documentId):
+@bp.route("/document/<filepath>")
+async def get_document(filepath):
     try:
-        document = retrieve_document(documentId)
+        document = retrieve_document(filepath)
         return jsonify(document), 200
     except Exception as e:
         logging.exception("Exception in /history/clear_messages")
@@ -1006,10 +1005,15 @@ async def generate_section_content(request_json):
     except Exception as e:
         raise e
 
-def retrieve_document(id):
+def retrieve_document(filepath):
     try:
         search_client = init_ai_search_client()
-        document = search_client.get_document(id)
+        search_query = f"filepath eq '{filepath}'"
+        # Execute the search query
+        results = search_client.search(search_query)
+
+        # Get the full_content of the first result
+        document = next(results)
         return document
     except Exception as e:
         logging.exception("Exception in retrieve_document")

frontend/src/pages/chat/Chat.tsx

@@ -783,7 +783,7 @@ const Chat = ({ type = ChatType.Browse }: Props) => {
   }, [showLoadingMessage, processMessages])
 
   const onShowCitation = (citation: Citation) => {
-    const path = `/#/document/${citation.id}`
+    const path = `/#/document/${citation.filepath}`;
 
     // Instead of navigating within the app, use window.open to open in a new tab
     const url = window.location.origin + path

frontend/src/pages/document/Document.tsx

@@ -5,6 +5,7 @@ import { useParams } from 'react-router-dom';
 // Define the interface for the document data
 interface DocumentData {
   content: string;
+  full_content: string;
 }
 
 const Document = (): JSX.Element => {
@@ -19,7 +20,6 @@ const Document = (): JSX.Element => {
       try {
         const response = await documentRead(id);
         const data = await response.json();
-
         setDocument(data);
       } catch (error) {
         console.error(error);
@@ -39,7 +39,7 @@ const Document = (): JSX.Element => {
       {isLoading ? ( // Step 4
         <p>Loading...</p>
       ) : document ? (
-        <p>{document.content}</p>
+        <p>{document.full_content}</p>
       ) : (
         <h1>Document not found. Please try again.</h1>
       )}

requirements-dev.txt

@@ -1,5 +1,5 @@
 -r requirements.txt
-azure-ai-formrecognizer==3.2.1
+azure-ai-documentintelligence==1.0.0b2
 Markdown==3.4.4
 requests==2.32.3
 tqdm==4.66.1
@@ -9,6 +9,8 @@ bs4==0.0.1
 urllib3==2.2.2
 pytest==7.4.0
 pytest-asyncio==0.23.2
+PyMuPDF==1.24.5
 azure-storage-blob
 chardet
-azure-keyvault-secrets
+azure-keyvault-secrets
+coverage

requirements.txt

@@ -9,5 +9,4 @@ quart==0.19.4
 uvicorn==0.24.0
 aiohttp==3.9.2
 gunicorn==20.1.0
-pydantic-settings==2.2.1
-azure-search-documents
+pydantic-settings==2.2.1