From 69a2bb576d5582f044241abf64a7b45298833015 Mon Sep 17 00:00:00 2001 From: Stan Girard Date: Sat, 5 Oct 2024 10:21:43 +0200 Subject: [PATCH 1/4] wip --- .../modules/sync/controller/sync_routes.py | 8 + .../sync/controller/zendesk_ask_token.py | 52 ++++++ .../sync/controller/zendesk_sync_routes.py | 161 ++++++++++++++++++ frontend/lib/api/sync/sync.ts | 11 ++ frontend/lib/api/sync/types.ts | 5 +- frontend/lib/api/sync/useSync.ts | 8 +- .../ConnectionCards/ConnectionCards.tsx | 8 +- 7 files changed, 249 insertions(+), 4 deletions(-) create mode 100644 backend/api/quivr_api/modules/sync/controller/zendesk_ask_token.py create mode 100644 backend/api/quivr_api/modules/sync/controller/zendesk_sync_routes.py diff --git a/backend/api/quivr_api/modules/sync/controller/sync_routes.py b/backend/api/quivr_api/modules/sync/controller/sync_routes.py index 3adcbe41b4bb..815eabeb0ac6 100644 --- a/backend/api/quivr_api/modules/sync/controller/sync_routes.py +++ b/backend/api/quivr_api/modules/sync/controller/sync_routes.py @@ -18,6 +18,7 @@ from quivr_api.modules.sync.controller.github_sync_routes import github_sync_router from quivr_api.modules.sync.controller.google_sync_routes import google_sync_router from quivr_api.modules.sync.controller.notion_sync_routes import notion_sync_router +from quivr_api.modules.sync.controller.zendesk_sync_routes import zendesk_sync_router from quivr_api.modules.sync.dto import SyncsDescription from quivr_api.modules.sync.dto.inputs import SyncsActiveInput, SyncsActiveUpdateInput from quivr_api.modules.sync.dto.outputs import AuthMethodEnum @@ -49,6 +50,7 @@ sync_router.include_router(github_sync_router) sync_router.include_router(dropbox_sync_router) sync_router.include_router(notion_sync_router) +sync_router.include_router(zendesk_sync_router) # Google sync description @@ -82,6 +84,12 @@ auth_method=AuthMethodEnum.URI_WITH_CALLBACK, ) +zendesk_sync = SyncsDescription( + name="Zendesk", + description="Sync your Zendesk with Quivr", + auth_method=AuthMethodEnum.URI_WITH_CALLBACK, +) + @sync_router.get( "/sync/all", diff --git a/backend/api/quivr_api/modules/sync/controller/zendesk_ask_token.py b/backend/api/quivr_api/modules/sync/controller/zendesk_ask_token.py new file mode 100644 index 000000000000..887e7fbf8922 --- /dev/null +++ b/backend/api/quivr_api/modules/sync/controller/zendesk_ask_token.py @@ -0,0 +1,52 @@ +zendeskAskTokenPage = """ + + + + Enter Zendesk API Token + + + +
+

Enter Your Zendesk API Token

+
+
+ +
+ +
+
+ + +""" diff --git a/backend/api/quivr_api/modules/sync/controller/zendesk_sync_routes.py b/backend/api/quivr_api/modules/sync/controller/zendesk_sync_routes.py new file mode 100644 index 000000000000..6acba635be06 --- /dev/null +++ b/backend/api/quivr_api/modules/sync/controller/zendesk_sync_routes.py @@ -0,0 +1,161 @@ + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import HTMLResponse +import random + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import HTMLResponse + +from quivr_api.logger import get_logger +from quivr_api.middlewares.auth import AuthBearer, get_current_user +from quivr_api.modules.sync.dto.inputs import ( + SyncsUserInput, + SyncsUserStatus, + SyncUserUpdateInput, +) +from quivr_api.modules.sync.service.sync_service import SyncService, SyncUserService +from quivr_api.modules.user.entity.user_identity import UserIdentity + +from quivr_api.logger import get_logger +from quivr_api.middlewares.auth import AuthBearer, get_current_user +from quivr_api.modules.sync.dto.inputs import ( + SyncsUserInput, + SyncsUserStatus, + SyncUserUpdateInput, +) +from quivr_api.modules.sync.service.sync_service import SyncService, SyncUserService +from quivr_api.modules.user.entity.user_identity import UserIdentity +from fastapi import Form + +from .successfull_connection import successfullConnectionPage + +# Initialize logger +logger = get_logger(__name__) + +# Initialize sync service +sync_service = SyncService() +sync_user_service = SyncUserService() + +# Initialize API router +zendesk_sync_router = APIRouter() + + # + + +@zendesk_sync_router.post( + "/sync/zendesk/authorize", + dependencies=[Depends(AuthBearer())], + tags=["Sync"], +) +def authorize_azure( + request: Request, name: str, current_user: UserIdentity = Depends(get_current_user) +): + """ + Authorize Azure sync for the current user. + + Args: + request (Request): The request object. + current_user (UserIdentity): The current authenticated user. + + Returns: + dict: A dictionary containing the authorization URL. + """ + + state = random.randint(100000, 999999) + sync_user_input = SyncsUserInput( + user_id=str(current_user.id), + name=name, + provider="Azure", + credentials={}, + state={}, + additional_data={}, + status=str(SyncsUserStatus.SYNCING), + ) + sync_user_service.create_sync_user(sync_user_input) + return {"authorization_url": f"http://stangirard.com:5050/sync/zendesk/enter-token?state={state}"} + +@zendesk_sync_router.get("/sync/zendesk/enter-token", tags=["Sync"]) +def enter_zendesk_token_page(request: Request): + """ + Serve the HTML page to enter the Zendesk API token. + """ + state = request.query_params.get("state", "") + zendeskAskTokenPage = f""" + + + + Enter Zendesk API Token + + + +
+

Enter Your Zendesk API Token

+
+
+ +
+ + +
+
+ + + """ + return zendeskAskTokenPage + +@zendesk_sync_router.post("/sync/zendesk/submit-token", tags=["Sync"]) +def submit_zendesk_token(api_token: str = Form(...), current_user: UserIdentity = Depends(get_current_user)): + """ + Handle the submission of the Zendesk API token. + + Args: + api_token (str): The API token provided by the user. + current_user (UserIdentity): The current authenticated user. + + Returns: + HTMLResponse: A success page. + """ + logger.debug(f"Received Zendesk API token for user: {current_user.id}") + + # Update the sync user with the provided Zendesk API token + sync_user_input = SyncUserUpdateInput( + email=current_user.email, + credentials={"api_token": api_token}, + status=str(SyncsUserStatus.SYNCED), + ) + sync_user_service.update_sync_user(current_user.id, {}, sync_user_input) + logger.info(f"Zendesk API token updated successfully for user: {current_user.id}") + + return HTMLResponse(successfullConnectionPage) \ No newline at end of file diff --git a/frontend/lib/api/sync/sync.ts b/frontend/lib/api/sync/sync.ts index c2a16004772e..f91f88a29421 100644 --- a/frontend/lib/api/sync/sync.ts +++ b/frontend/lib/api/sync/sync.ts @@ -37,6 +37,17 @@ export const syncSharepoint = async ( ).data; }; +export const syncZendesk = async ( + name: string, + axiosInstance: AxiosInstance +): Promise<{ authorization_url: string }> => { + return ( + await axiosInstance.post<{ authorization_url: string }>( + `/sync/zendesk/authorize?name=${name}` + ) + ).data; +}; + export const syncDropbox = async ( name: string, axiosInstance: AxiosInstance diff --git a/frontend/lib/api/sync/types.ts b/frontend/lib/api/sync/types.ts index 75fae2764adc..4ac1daba3fe0 100644 --- a/frontend/lib/api/sync/types.ts +++ b/frontend/lib/api/sync/types.ts @@ -1,11 +1,12 @@ -export type Provider = "Google" | "Azure" | "DropBox" | "Notion" | "GitHub"; +export type Provider = "Google" | "Azure" | "DropBox" | "Notion" | "GitHub" | "Zendesk"; export type Integration = | "Google Drive" | "Share Point" | "Dropbox" | "Notion" - | "GitHub"; + | "GitHub" + | "Zendesk" export type SyncStatus = "SYNCING" | "SYNCED" | "ERROR" | "REMOVED"; diff --git a/frontend/lib/api/sync/useSync.ts b/frontend/lib/api/sync/useSync.ts index 141ce71c97d3..ef0fb7f0d5fa 100644 --- a/frontend/lib/api/sync/useSync.ts +++ b/frontend/lib/api/sync/useSync.ts @@ -13,7 +13,8 @@ import { syncGoogleDrive, syncNotion, syncSharepoint, - updateActiveSync + syncZendesk, + updateActiveSync, } from "./sync"; import { Integration, OpenedConnection, Provider } from "./types"; @@ -32,6 +33,8 @@ export const useSync = () => { "https://quivr-cms.s3.eu-west-3.amazonaws.com/Notion_app_logo_004168672c.png", GitHub: "https://quivr-cms.s3.eu-west-3.amazonaws.com/dropbox_dce4f3d753.png", + Zendesk: + "https://quivr-cms.s3.eu-west-3.amazonaws.com/dropbox_dce4f3d753.png", }; const integrationIconUrls: Record = { @@ -45,6 +48,8 @@ export const useSync = () => { "https://quivr-cms.s3.eu-west-3.amazonaws.com/Notion_app_logo_004168672c.png", GitHub: "https://quivr-cms.s3.eu-west-3.amazonaws.com/dropbox_dce4f3d753.png", + Zendesk: + "https://quivr-cms.s3.eu-west-3.amazonaws.com/dropbox_dce4f3d753.png", }; const getActiveSyncsForBrain = async (brainId: string) => { @@ -59,6 +64,7 @@ export const useSync = () => { syncSharepoint: async (name: string) => syncSharepoint(name, axiosInstance), syncDropbox: async (name: string) => syncDropbox(name, axiosInstance), syncNotion: async (name: string) => syncNotion(name, axiosInstance), + syncZendesk: async (name: string) => syncZendesk(name, axiosInstance), getUserSyncs: async () => getUserSyncs(axiosInstance), getSyncFiles: async (userSyncId: number, folderId?: string) => getSyncFiles(axiosInstance, userSyncId, folderId), diff --git a/frontend/lib/components/ConnectionCards/ConnectionCards.tsx b/frontend/lib/components/ConnectionCards/ConnectionCards.tsx index ae81175a8302..a7c2c62ae1eb 100644 --- a/frontend/lib/components/ConnectionCards/ConnectionCards.tsx +++ b/frontend/lib/components/ConnectionCards/ConnectionCards.tsx @@ -10,7 +10,7 @@ interface ConnectionCardsProps { export const ConnectionCards = ({ fromAddKnowledge, }: ConnectionCardsProps): JSX.Element => { - const { syncGoogleDrive, syncSharepoint, syncDropbox } = + const { syncGoogleDrive, syncSharepoint, syncDropbox, syncZendesk } = useSync(); return ( @@ -37,6 +37,12 @@ export const ConnectionCards = ({ fromAddKnowledge={fromAddKnowledge} oneAccountLimitation={true} /> */} + syncZendesk(name)} + fromAddKnowledge={fromAddKnowledge} + /> Date: Wed, 9 Oct 2024 11:52:09 +0200 Subject: [PATCH 2/4] add zen desk list & listbyid and download --- .../modules/sync/controller/sync_routes.py | 2 +- .../sync/controller/zendesk_sync_routes.py | 135 +++++++++----- .../modules/sync/repository/sync_user.py | 9 + .../api/quivr_api/modules/sync/utils/sync.py | 164 ++++++++++++++++++ .../quivr_api/modules/sync/utils/syncutils.py | 3 +- backend/requirements.lock | 42 +++-- backend/worker/quivr_worker/syncs/utils.py | 2 + 7 files changed, 298 insertions(+), 59 deletions(-) diff --git a/backend/api/quivr_api/modules/sync/controller/sync_routes.py b/backend/api/quivr_api/modules/sync/controller/sync_routes.py index 815eabeb0ac6..ba983389bc61 100644 --- a/backend/api/quivr_api/modules/sync/controller/sync_routes.py +++ b/backend/api/quivr_api/modules/sync/controller/sync_routes.py @@ -108,7 +108,7 @@ async def get_syncs(current_user: UserIdentity = Depends(get_current_user)): List[SyncsDescription]: A list of available sync descriptions. """ logger.debug(f"Fetching all sync descriptions for user: {current_user.id}") - return [google_sync, azure_sync, dropbox_sync, notion_sync] + return [google_sync, azure_sync, dropbox_sync, notion_sync, zendesk_sync] @sync_router.get( diff --git a/backend/api/quivr_api/modules/sync/controller/zendesk_sync_routes.py b/backend/api/quivr_api/modules/sync/controller/zendesk_sync_routes.py index 6acba635be06..90bf6cd838f9 100644 --- a/backend/api/quivr_api/modules/sync/controller/zendesk_sync_routes.py +++ b/backend/api/quivr_api/modules/sync/controller/zendesk_sync_routes.py @@ -1,9 +1,6 @@ - -from fastapi import APIRouter, Depends, Request -from fastapi.responses import HTMLResponse import random -from fastapi import APIRouter, Depends, Request +from fastapi import APIRouter, Depends, Form, HTTPException, Request from fastapi.responses import HTMLResponse from quivr_api.logger import get_logger @@ -16,17 +13,6 @@ from quivr_api.modules.sync.service.sync_service import SyncService, SyncUserService from quivr_api.modules.user.entity.user_identity import UserIdentity -from quivr_api.logger import get_logger -from quivr_api.middlewares.auth import AuthBearer, get_current_user -from quivr_api.modules.sync.dto.inputs import ( - SyncsUserInput, - SyncsUserStatus, - SyncUserUpdateInput, -) -from quivr_api.modules.sync.service.sync_service import SyncService, SyncUserService -from quivr_api.modules.user.entity.user_identity import UserIdentity -from fastapi import Form - from .successfull_connection import successfullConnectionPage # Initialize logger @@ -39,7 +25,7 @@ # Initialize API router zendesk_sync_router = APIRouter() - # +# @zendesk_sync_router.post( @@ -47,11 +33,11 @@ dependencies=[Depends(AuthBearer())], tags=["Sync"], ) -def authorize_azure( +def authorize_zendesk( request: Request, name: str, current_user: UserIdentity = Depends(get_current_user) ): """ - Authorize Azure sync for the current user. + Authorize Zendesk sync for the current user. Args: request (Request): The request object. @@ -61,30 +47,33 @@ def authorize_azure( dict: A dictionary containing the authorization URL. """ - state = random.randint(100000, 999999) + state = str(current_user.email) + "," + str(random.randint(100000, 999999)) sync_user_input = SyncsUserInput( user_id=str(current_user.id), name=name, - provider="Azure", + provider="Zendesk", credentials={}, - state={}, + state={"state": state.split(",")[1]}, additional_data={}, status=str(SyncsUserStatus.SYNCING), ) sync_user_service.create_sync_user(sync_user_input) - return {"authorization_url": f"http://stangirard.com:5050/sync/zendesk/enter-token?state={state}"} + return { + "authorization_url": f"http://localhost:5050/sync/zendesk/enter-token?state={state}" + } + @zendesk_sync_router.get("/sync/zendesk/enter-token", tags=["Sync"]) def enter_zendesk_token_page(request: Request): """ - Serve the HTML page to enter the Zendesk API token. + Serve the HTML page to enter the Zendesk API token and domain name. """ state = request.query_params.get("state", "") zendeskAskTokenPage = f""" - Enter Zendesk API Token + Enter Zendesk API Token and Domain
-

Enter Your Zendesk API Token

+

Enter Your Zendesk API Token and Domain

+
+ +
+
+ + .zendesk.com +
@@ -133,10 +152,16 @@ def enter_zendesk_token_page(request: Request): """ - return zendeskAskTokenPage + return HTMLResponse(content=zendeskAskTokenPage, status_code=200) + @zendesk_sync_router.post("/sync/zendesk/submit-token", tags=["Sync"]) -def submit_zendesk_token(api_token: str = Form(...), current_user: UserIdentity = Depends(get_current_user)): +def submit_zendesk_token( + api_token: str = Form(...), + sub_domain_name: str = Form(...), + email: str = Form(...), + state: str = Form(...), +): """ Handle the submission of the Zendesk API token. @@ -147,15 +172,35 @@ def submit_zendesk_token(api_token: str = Form(...), current_user: UserIdentity Returns: HTMLResponse: A success page. """ - logger.debug(f"Received Zendesk API token for user: {current_user.id}") + user_email, sync_state = state.split(",") + state_dict = {"state": sync_state} + logger.debug(f"Handling OAuth2 callback for user with state: {state}") + sync_user_state = sync_user_service.get_sync_user_by_state(state_dict) + logger.info(f"Retrieved sync user state: {sync_user_state}") + if not sync_user_state or state_dict != sync_user_state.state: + logger.error("Invalid state parameter") + raise HTTPException(status_code=400, detail="Invalid state parameter") + + logger.debug( + f"Received Zendesk API token and sub domain name for user: {sync_user_state.user_id}" + ) + assert email is not None, "User email is None" # Update the sync user with the provided Zendesk API token sync_user_input = SyncUserUpdateInput( - email=current_user.email, - credentials={"api_token": api_token}, + email=email, + credentials={ + "api_token": api_token, + "sub_domain_name": sub_domain_name, + "email": email, + }, status=str(SyncsUserStatus.SYNCED), ) - sync_user_service.update_sync_user(current_user.id, {}, sync_user_input) - logger.info(f"Zendesk API token updated successfully for user: {current_user.id}") + sync_user_service.update_sync_user( + sync_user_state.user_id, state_dict, sync_user_input + ) + logger.info( + f"Zendesk API token updated successfully for user: {sync_user_state.user_id}" + ) - return HTMLResponse(successfullConnectionPage) \ No newline at end of file + return HTMLResponse(successfullConnectionPage) diff --git a/backend/api/quivr_api/modules/sync/repository/sync_user.py b/backend/api/quivr_api/modules/sync/repository/sync_user.py index 09ff5007d7b4..8ad0c7359c61 100644 --- a/backend/api/quivr_api/modules/sync/repository/sync_user.py +++ b/backend/api/quivr_api/modules/sync/repository/sync_user.py @@ -17,6 +17,7 @@ GitHubSync, GoogleDriveSync, NotionSync, + ZendeskSync, ) logger = get_logger(__name__) @@ -287,6 +288,14 @@ async def get_files_folder_user_sync( sync_user["credentials"], folder_id if folder_id else "", recursive ) } + elif provider == "zendesk": + logger.info("Getting files for Zendesk sync") + sync = ZendeskSync() + return { + "files": sync.get_files( + sync_user["credentials"], folder_id if folder_id else "", recursive + ) + } else: logger.warning( diff --git a/backend/api/quivr_api/modules/sync/utils/sync.py b/backend/api/quivr_api/modules/sync/utils/sync.py index a60ccb0aa75f..bf837a917186 100644 --- a/backend/api/quivr_api/modules/sync/utils/sync.py +++ b/backend/api/quivr_api/modules/sync/utils/sync.py @@ -1218,3 +1218,167 @@ def fetch_files(endpoint, headers): logger.info(f"GitHub repository files retrieved successfully: {len(files)}") return files + + +class ZendeskSync(BaseSync): + name = "Zendesk" + lower_name = "zendesk" + datetime_format = "%Y-%m-%dT%H:%M:%SZ" + + def get_zendesk_api_url(self, credentials: Dict): + return f"https://{credentials['sub_domain_name']}.zendesk.com/api/v2" + + def get_files( + self, credentials: Dict, folder_id: str | None = "", recursive: bool = False + ) -> List[SyncFile]: + """ + Retrieve files from Zendesk. + + Args: + credentials (Dict): Zendesk API credentials. + folder_path (str): Not used for Zendesk, but kept for consistency with other sync methods. + recursive (bool): Not used for Zendesk, but kept for consistency with other sync methods. + + Returns: + List[SyncFile]: List of SyncFile objects representing Zendesk tickets. + """ + + logger.info(f"Retrieving Zendesk tickets with credentials: {credentials}") + url = f"{self.get_zendesk_api_url(credentials)}/tickets.json" + headers = { + "Content-Type": "application/json", + } + email_address = f"{credentials['email']}/token" + api_token = credentials["api_token"] + auth = (email_address, api_token) + + response = requests.get(url, auth=auth, headers=headers) + response.raise_for_status() + + tickets = response.json().get("tickets", []) + logger.debug( + f"Retrieved {len(tickets)} tickets from Zendesk, example ticket: {tickets[0] if tickets else 'No tickets'}" + ) + files = [] + + for ticket in tickets: + file_data = SyncFile( + name=f"{ticket['subject']}.json", + id=str(ticket["id"]), + is_folder=False, + last_modified=ticket["updated_at"], + mime_type=".json", + web_view_link=f"{self.get_zendesk_api_url(credentials)}/tickets/{ticket['id']}", + size=len(json.dumps(ticket)), + ) + files.append(file_data) + + logger.info(f"Zendesk tickets retrieved successfully: {len(files)}") + return files + + async def aget_files( + self, + credentials: Dict, + folder_id: str | None = "", + recursive: bool = False, + sync_user_id: int | None = None, + ) -> List[SyncFile]: + return self.get_files(credentials, folder_id, recursive) + + def get_files_by_id( + self, + credentials: Dict, + file_ids: List[str], + ) -> List[SyncFile]: + """ + Retrieve specific Zendesk tickets by their IDs. + + Args: + credentials (Dict): Zendesk API credentials. + file_ids (List[str]): List of ticket IDs to retrieve. + + Returns: + List[SyncFile]: List of SyncFile objects representing the specified Zendesk tickets. + """ + logger.info(f"Retrieving specific Zendesk tickets with IDs: {file_ids}") + url = f"{self.get_zendesk_api_url(credentials)}/tickets/show_many.json?ids={','.join(file_ids)}" + headers = { + "Content-Type": "application/json", + } + email_address = f"{credentials['email']}/token" + api_token = credentials["api_token"] + auth = (email_address, api_token) + + response = requests.get(url, auth=auth, headers=headers) + response.raise_for_status() + + tickets = response.json().get("tickets", []) + logger.debug(f"Retrieved {len(tickets)} tickets from Zendesk") + + files = [] + for ticket in tickets: + file_data = SyncFile( + name=f"{ticket['subject']}.json", + id=str(ticket["id"]), + is_folder=False, + last_modified=ticket["updated_at"], + mime_type=".json", + web_view_link=f"{self.get_zendesk_api_url(credentials)}/tickets/{ticket['id']}", + size=len(json.dumps(ticket)), + ) + files.append(file_data) + + logger.info(f"Zendesk tickets retrieved successfully: {len(files)}") + return files + + async def aget_files_by_id( + self, credentials: Dict, file_ids: List[str] + ) -> List[SyncFile]: + return self.get_files_by_id(credentials, file_ids) + + def download_file( + self, credentials: Dict, file: SyncFile + ) -> Dict[str, Union[str, BytesIO]]: + """ + Download a specific Zendesk ticket as a JSON file. + + Args: + credentials (Dict): Zendesk API credentials. + file_id (str): ID of the ticket to download. + file_name (str): Name of the file to be downloaded. + + Returns: + BytesIO: A BytesIO object containing the JSON data of the ticket. + """ + logger.info(f"Downloading Zendesk ticket with ID: {file.id}") + url = f"{self.get_zendesk_api_url(credentials)}/tickets/{file.id}/comments.json" + headers = { + "Content-Type": "application/json", + } + email_address = f"{credentials['email']}/token" + api_token = credentials["api_token"] + auth = (email_address, api_token) + + response = requests.get(url, auth=auth, headers=headers) + + ticket_data = response.json().get("comments", {}) + + # Convert the ticket data to a JSON string + json_data = json.dumps(ticket_data, indent=2) + + # Create a BytesIO object from the JSON string + file_content = BytesIO(json_data.encode("utf-8")) + + return { + "file_name": f"{file.name.rsplit('.', 1)[0]}.txt", + "content": file_content, + } + + async def adownload_file( + self, credentials: Dict, file: SyncFile + ) -> Dict[str, Union[str, BytesIO]]: + return self.download_file(credentials, file) + + def check_and_refresh_access_token(self, credentials: Dict) -> Dict: + # not needed for Zendesk + return credentials diff --git a/backend/api/quivr_api/modules/sync/utils/syncutils.py b/backend/api/quivr_api/modules/sync/utils/syncutils.py index 5fe9f53105b0..7594e1bf7f4b 100644 --- a/backend/api/quivr_api/modules/sync/utils/syncutils.py +++ b/backend/api/quivr_api/modules/sync/utils/syncutils.py @@ -132,7 +132,7 @@ async def download_file( ) extension = os.path.splitext(file_name)[-1].lower() dfile = DownloadedSyncFile( - file_name=file_name, + file_name=file_name.encode("ascii", errors="ignore").decode("ascii"), file_data=file_data, extension=extension, ) @@ -165,6 +165,7 @@ async def process_sync_file( ".xlsx", ".pptx", ".doc", + ".json", ]: raise ValueError(f"Incompatible file extension for {downloaded_file}") diff --git a/backend/requirements.lock b/backend/requirements.lock index 7bb40f61eaef..e2761ec741f3 100644 --- a/backend/requirements.lock +++ b/backend/requirements.lock @@ -611,6 +611,34 @@ numpy==1.26.3 # via torchvision # via transformers # via unstructured +nvidia-cublas-cu12==12.1.3.1 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via nvidia-cudnn-cu12 + # via nvidia-cusolver-cu12 + # via torch +nvidia-cuda-cupti-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cuda-nvrtc-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cuda-runtime-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cudnn-cu12==9.1.0.70 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cufft-cu12==11.0.2.54 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-curand-cu12==10.3.2.106 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cusolver-cu12==11.4.5.107 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cusparse-cu12==12.1.0.106 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via nvidia-cusolver-cu12 + # via torch +nvidia-nccl-cu12==2.20.5 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-nvjitlink-cu12==12.6.77 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via nvidia-cusolver-cu12 + # via nvidia-cusparse-cu12 +nvidia-nvtx-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch oauthlib==3.2.2 # via requests-oauthlib olefile==0.47 @@ -1032,23 +1060,13 @@ tokenizers==0.19.1 # via cohere # via litellm # via transformers -torch==2.4.0 ; platform_machine != 'x86_64' +torch==2.4.0 # via effdet # via quivr-worker # via timm # via torchvision # via unstructured-inference -torch==2.4.0+cpu ; platform_machine == 'x86_64' - # via effdet - # via quivr-worker - # via timm - # via torchvision - # via unstructured-inference -torchvision==0.19.0 ; platform_machine != 'x86_64' - # via effdet - # via quivr-worker - # via timm -torchvision==0.19.0+cpu ; platform_machine == 'x86_64' +torchvision==0.19.0 # via effdet # via quivr-worker # via timm diff --git a/backend/worker/quivr_worker/syncs/utils.py b/backend/worker/quivr_worker/syncs/utils.py index bbc3c75f8588..7911af61d9b9 100644 --- a/backend/worker/quivr_worker/syncs/utils.py +++ b/backend/worker/quivr_worker/syncs/utils.py @@ -23,6 +23,7 @@ GitHubSync, GoogleDriveSync, NotionSync, + ZendeskSync, ) from quivr_api.modules.sync.utils.syncutils import SyncUtils from sqlalchemy.ext.asyncio import AsyncEngine @@ -70,6 +71,7 @@ async def build_syncs_utils( "notion", NotionSync(notion_service=notion_service), ), # Fixed duplicate "github" key + ("zendesk", ZendeskSync()), ]: provider_sync_util = SyncUtils( sync_user_service=deps.sync_user_service, From 874a68847255458841f82a66810bea0e929498cc Mon Sep 17 00:00:00 2001 From: Stan Girard Date: Wed, 9 Oct 2024 12:40:29 +0200 Subject: [PATCH 3/4] feat: Update source document handling in rag_service/utils.py Refactor the `generate_source` function in `rag_service/utils.py` to improve the handling of source documents. This includes logging additional information about source documents and integrating with Zendesk for certain document types. The changes enhance the reliability and functionality of the source document generation process. Co-authored-by: Stan Girard --- backend/api/quivr_api/modules/rag_service/utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/api/quivr_api/modules/rag_service/utils.py b/backend/api/quivr_api/modules/rag_service/utils.py index afc12082eac8..176b7eb7c9dd 100644 --- a/backend/api/quivr_api/modules/rag_service/utils.py +++ b/backend/api/quivr_api/modules/rag_service/utils.py @@ -33,6 +33,7 @@ async def generate_source( # Get source documents from the result, default to an empty list if not found # If source documents exist + logger.info(f"Source documents: {source_documents}") if source_documents: logger.debug(f"Citations {citations}") for index, doc in enumerate(source_documents): @@ -48,6 +49,7 @@ async def generate_source( "original_file_name" in doc.metadata and doc.metadata["original_file_name"] is not None and doc.metadata["original_file_name"].startswith("http") + and doc.metadata["integration"] == "" ) # Determine the name based on whether it's a URL or a file @@ -63,6 +65,8 @@ async def generate_source( # Determine the source URL based on whether it's a URL or a file if is_url: source_url = doc.metadata["original_file_name"] + elif doc.metadata["integration"] != "": + logger.info(f"Integration: {doc.metadata['integration']}") else: # Check if the URL has already been generated try: @@ -87,8 +91,12 @@ async def generate_source( except Exception as e: logger.error(f"Error generating file signed URL: {e}") continue - + + logger.info(f"Metadata: {doc.metadata}") # Append a new Sources object to the list + if doc.metadata["integration"] == "Zendesk": + logger.error(f"Zendesk integration: {doc.metadata['integration']}") + source_url = doc.metadata["integration_link"] sources_list.append( Sources( name=name, From 5fbea4f1e52b348686c55dbb476528ced190340a Mon Sep 17 00:00:00 2001 From: Stan Girard Date: Wed, 9 Oct 2024 16:03:26 +0200 Subject: [PATCH 4/4] refactor: Update Zendesk integration for source document handling --- frontend/lib/api/sync/useSync.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/lib/api/sync/useSync.ts b/frontend/lib/api/sync/useSync.ts index ef0fb7f0d5fa..ff187a776066 100644 --- a/frontend/lib/api/sync/useSync.ts +++ b/frontend/lib/api/sync/useSync.ts @@ -34,7 +34,7 @@ export const useSync = () => { GitHub: "https://quivr-cms.s3.eu-west-3.amazonaws.com/dropbox_dce4f3d753.png", Zendesk: - "https://quivr-cms.s3.eu-west-3.amazonaws.com/dropbox_dce4f3d753.png", + "https://quivr-cms.s3.eu-west-3.amazonaws.com/zendesk_c39745607c.png", }; const integrationIconUrls: Record = { @@ -49,7 +49,7 @@ export const useSync = () => { GitHub: "https://quivr-cms.s3.eu-west-3.amazonaws.com/dropbox_dce4f3d753.png", Zendesk: - "https://quivr-cms.s3.eu-west-3.amazonaws.com/dropbox_dce4f3d753.png", + "https://quivr-cms.s3.eu-west-3.amazonaws.com/zendesk_c39745607c.png", }; const getActiveSyncsForBrain = async (brainId: string) => {