diff --git a/backend/api/quivr_api/modules/rag_service/utils.py b/backend/api/quivr_api/modules/rag_service/utils.py index afc12082eac8..176b7eb7c9dd 100644 --- a/backend/api/quivr_api/modules/rag_service/utils.py +++ b/backend/api/quivr_api/modules/rag_service/utils.py @@ -33,6 +33,7 @@ async def generate_source( # Get source documents from the result, default to an empty list if not found # If source documents exist + logger.info(f"Source documents: {source_documents}") if source_documents: logger.debug(f"Citations {citations}") for index, doc in enumerate(source_documents): @@ -48,6 +49,7 @@ async def generate_source( "original_file_name" in doc.metadata and doc.metadata["original_file_name"] is not None and doc.metadata["original_file_name"].startswith("http") + and doc.metadata["integration"] == "" ) # Determine the name based on whether it's a URL or a file @@ -63,6 +65,8 @@ async def generate_source( # Determine the source URL based on whether it's a URL or a file if is_url: source_url = doc.metadata["original_file_name"] + elif doc.metadata["integration"] != "": + logger.info(f"Integration: {doc.metadata['integration']}") else: # Check if the URL has already been generated try: @@ -87,8 +91,12 @@ async def generate_source( except Exception as e: logger.error(f"Error generating file signed URL: {e}") continue - + + logger.info(f"Metadata: {doc.metadata}") # Append a new Sources object to the list + if doc.metadata["integration"] == "Zendesk": + logger.error(f"Zendesk integration: {doc.metadata['integration']}") + source_url = doc.metadata["integration_link"] sources_list.append( Sources( name=name, diff --git a/backend/api/quivr_api/modules/sync/controller/sync_routes.py b/backend/api/quivr_api/modules/sync/controller/sync_routes.py index 3adcbe41b4bb..ba983389bc61 100644 --- a/backend/api/quivr_api/modules/sync/controller/sync_routes.py +++ b/backend/api/quivr_api/modules/sync/controller/sync_routes.py @@ -18,6 +18,7 @@ from quivr_api.modules.sync.controller.github_sync_routes import github_sync_router from quivr_api.modules.sync.controller.google_sync_routes import google_sync_router from quivr_api.modules.sync.controller.notion_sync_routes import notion_sync_router +from quivr_api.modules.sync.controller.zendesk_sync_routes import zendesk_sync_router from quivr_api.modules.sync.dto import SyncsDescription from quivr_api.modules.sync.dto.inputs import SyncsActiveInput, SyncsActiveUpdateInput from quivr_api.modules.sync.dto.outputs import AuthMethodEnum @@ -49,6 +50,7 @@ sync_router.include_router(github_sync_router) sync_router.include_router(dropbox_sync_router) sync_router.include_router(notion_sync_router) +sync_router.include_router(zendesk_sync_router) # Google sync description @@ -82,6 +84,12 @@ auth_method=AuthMethodEnum.URI_WITH_CALLBACK, ) +zendesk_sync = SyncsDescription( + name="Zendesk", + description="Sync your Zendesk with Quivr", + auth_method=AuthMethodEnum.URI_WITH_CALLBACK, +) + @sync_router.get( "/sync/all", @@ -100,7 +108,7 @@ async def get_syncs(current_user: UserIdentity = Depends(get_current_user)): List[SyncsDescription]: A list of available sync descriptions. """ logger.debug(f"Fetching all sync descriptions for user: {current_user.id}") - return [google_sync, azure_sync, dropbox_sync, notion_sync] + return [google_sync, azure_sync, dropbox_sync, notion_sync, zendesk_sync] @sync_router.get( diff --git a/backend/api/quivr_api/modules/sync/controller/zendesk_ask_token.py b/backend/api/quivr_api/modules/sync/controller/zendesk_ask_token.py new file mode 100644 index 000000000000..887e7fbf8922 --- /dev/null +++ b/backend/api/quivr_api/modules/sync/controller/zendesk_ask_token.py @@ -0,0 +1,52 @@ +zendeskAskTokenPage = """ + + + + Enter Zendesk API Token + + + +
+

Enter Your Zendesk API Token

+
+
+ +
+ +
+
+ + +""" diff --git a/backend/api/quivr_api/modules/sync/controller/zendesk_sync_routes.py b/backend/api/quivr_api/modules/sync/controller/zendesk_sync_routes.py new file mode 100644 index 000000000000..90bf6cd838f9 --- /dev/null +++ b/backend/api/quivr_api/modules/sync/controller/zendesk_sync_routes.py @@ -0,0 +1,206 @@ +import random + +from fastapi import APIRouter, Depends, Form, HTTPException, Request +from fastapi.responses import HTMLResponse + +from quivr_api.logger import get_logger +from quivr_api.middlewares.auth import AuthBearer, get_current_user +from quivr_api.modules.sync.dto.inputs import ( + SyncsUserInput, + SyncsUserStatus, + SyncUserUpdateInput, +) +from quivr_api.modules.sync.service.sync_service import SyncService, SyncUserService +from quivr_api.modules.user.entity.user_identity import UserIdentity + +from .successfull_connection import successfullConnectionPage + +# Initialize logger +logger = get_logger(__name__) + +# Initialize sync service +sync_service = SyncService() +sync_user_service = SyncUserService() + +# Initialize API router +zendesk_sync_router = APIRouter() + +# + + +@zendesk_sync_router.post( + "/sync/zendesk/authorize", + dependencies=[Depends(AuthBearer())], + tags=["Sync"], +) +def authorize_zendesk( + request: Request, name: str, current_user: UserIdentity = Depends(get_current_user) +): + """ + Authorize Zendesk sync for the current user. + + Args: + request (Request): The request object. + current_user (UserIdentity): The current authenticated user. + + Returns: + dict: A dictionary containing the authorization URL. + """ + + state = str(current_user.email) + "," + str(random.randint(100000, 999999)) + sync_user_input = SyncsUserInput( + user_id=str(current_user.id), + name=name, + provider="Zendesk", + credentials={}, + state={"state": state.split(",")[1]}, + additional_data={}, + status=str(SyncsUserStatus.SYNCING), + ) + sync_user_service.create_sync_user(sync_user_input) + return { + "authorization_url": f"http://localhost:5050/sync/zendesk/enter-token?state={state}" + } + + +@zendesk_sync_router.get("/sync/zendesk/enter-token", tags=["Sync"]) +def enter_zendesk_token_page(request: Request): + """ + Serve the HTML page to enter the Zendesk API token and domain name. + """ + state = request.query_params.get("state", "") + zendeskAskTokenPage = f""" + + + + Enter Zendesk API Token and Domain + + + +
+

Enter Your Zendesk API Token and Domain

+
+
+ +
+
+ + .zendesk.com +
+
+ +
+ + +
+
+ + + """ + return HTMLResponse(content=zendeskAskTokenPage, status_code=200) + + +@zendesk_sync_router.post("/sync/zendesk/submit-token", tags=["Sync"]) +def submit_zendesk_token( + api_token: str = Form(...), + sub_domain_name: str = Form(...), + email: str = Form(...), + state: str = Form(...), +): + """ + Handle the submission of the Zendesk API token. + + Args: + api_token (str): The API token provided by the user. + current_user (UserIdentity): The current authenticated user. + + Returns: + HTMLResponse: A success page. + """ + user_email, sync_state = state.split(",") + state_dict = {"state": sync_state} + logger.debug(f"Handling OAuth2 callback for user with state: {state}") + sync_user_state = sync_user_service.get_sync_user_by_state(state_dict) + logger.info(f"Retrieved sync user state: {sync_user_state}") + if not sync_user_state or state_dict != sync_user_state.state: + logger.error("Invalid state parameter") + raise HTTPException(status_code=400, detail="Invalid state parameter") + + logger.debug( + f"Received Zendesk API token and sub domain name for user: {sync_user_state.user_id}" + ) + assert email is not None, "User email is None" + + # Update the sync user with the provided Zendesk API token + sync_user_input = SyncUserUpdateInput( + email=email, + credentials={ + "api_token": api_token, + "sub_domain_name": sub_domain_name, + "email": email, + }, + status=str(SyncsUserStatus.SYNCED), + ) + sync_user_service.update_sync_user( + sync_user_state.user_id, state_dict, sync_user_input + ) + logger.info( + f"Zendesk API token updated successfully for user: {sync_user_state.user_id}" + ) + + return HTMLResponse(successfullConnectionPage) diff --git a/backend/api/quivr_api/modules/sync/repository/sync_user.py b/backend/api/quivr_api/modules/sync/repository/sync_user.py index 09ff5007d7b4..8ad0c7359c61 100644 --- a/backend/api/quivr_api/modules/sync/repository/sync_user.py +++ b/backend/api/quivr_api/modules/sync/repository/sync_user.py @@ -17,6 +17,7 @@ GitHubSync, GoogleDriveSync, NotionSync, + ZendeskSync, ) logger = get_logger(__name__) @@ -287,6 +288,14 @@ async def get_files_folder_user_sync( sync_user["credentials"], folder_id if folder_id else "", recursive ) } + elif provider == "zendesk": + logger.info("Getting files for Zendesk sync") + sync = ZendeskSync() + return { + "files": sync.get_files( + sync_user["credentials"], folder_id if folder_id else "", recursive + ) + } else: logger.warning( diff --git a/backend/api/quivr_api/modules/sync/utils/sync.py b/backend/api/quivr_api/modules/sync/utils/sync.py index a60ccb0aa75f..bf837a917186 100644 --- a/backend/api/quivr_api/modules/sync/utils/sync.py +++ b/backend/api/quivr_api/modules/sync/utils/sync.py @@ -1218,3 +1218,167 @@ def fetch_files(endpoint, headers): logger.info(f"GitHub repository files retrieved successfully: {len(files)}") return files + + +class ZendeskSync(BaseSync): + name = "Zendesk" + lower_name = "zendesk" + datetime_format = "%Y-%m-%dT%H:%M:%SZ" + + def get_zendesk_api_url(self, credentials: Dict): + return f"https://{credentials['sub_domain_name']}.zendesk.com/api/v2" + + def get_files( + self, credentials: Dict, folder_id: str | None = "", recursive: bool = False + ) -> List[SyncFile]: + """ + Retrieve files from Zendesk. + + Args: + credentials (Dict): Zendesk API credentials. + folder_path (str): Not used for Zendesk, but kept for consistency with other sync methods. + recursive (bool): Not used for Zendesk, but kept for consistency with other sync methods. + + Returns: + List[SyncFile]: List of SyncFile objects representing Zendesk tickets. + """ + + logger.info(f"Retrieving Zendesk tickets with credentials: {credentials}") + url = f"{self.get_zendesk_api_url(credentials)}/tickets.json" + headers = { + "Content-Type": "application/json", + } + email_address = f"{credentials['email']}/token" + api_token = credentials["api_token"] + auth = (email_address, api_token) + + response = requests.get(url, auth=auth, headers=headers) + response.raise_for_status() + + tickets = response.json().get("tickets", []) + logger.debug( + f"Retrieved {len(tickets)} tickets from Zendesk, example ticket: {tickets[0] if tickets else 'No tickets'}" + ) + files = [] + + for ticket in tickets: + file_data = SyncFile( + name=f"{ticket['subject']}.json", + id=str(ticket["id"]), + is_folder=False, + last_modified=ticket["updated_at"], + mime_type=".json", + web_view_link=f"{self.get_zendesk_api_url(credentials)}/tickets/{ticket['id']}", + size=len(json.dumps(ticket)), + ) + files.append(file_data) + + logger.info(f"Zendesk tickets retrieved successfully: {len(files)}") + return files + + async def aget_files( + self, + credentials: Dict, + folder_id: str | None = "", + recursive: bool = False, + sync_user_id: int | None = None, + ) -> List[SyncFile]: + return self.get_files(credentials, folder_id, recursive) + + def get_files_by_id( + self, + credentials: Dict, + file_ids: List[str], + ) -> List[SyncFile]: + """ + Retrieve specific Zendesk tickets by their IDs. + + Args: + credentials (Dict): Zendesk API credentials. + file_ids (List[str]): List of ticket IDs to retrieve. + + Returns: + List[SyncFile]: List of SyncFile objects representing the specified Zendesk tickets. + """ + logger.info(f"Retrieving specific Zendesk tickets with IDs: {file_ids}") + url = f"{self.get_zendesk_api_url(credentials)}/tickets/show_many.json?ids={','.join(file_ids)}" + headers = { + "Content-Type": "application/json", + } + email_address = f"{credentials['email']}/token" + api_token = credentials["api_token"] + auth = (email_address, api_token) + + response = requests.get(url, auth=auth, headers=headers) + response.raise_for_status() + + tickets = response.json().get("tickets", []) + logger.debug(f"Retrieved {len(tickets)} tickets from Zendesk") + + files = [] + for ticket in tickets: + file_data = SyncFile( + name=f"{ticket['subject']}.json", + id=str(ticket["id"]), + is_folder=False, + last_modified=ticket["updated_at"], + mime_type=".json", + web_view_link=f"{self.get_zendesk_api_url(credentials)}/tickets/{ticket['id']}", + size=len(json.dumps(ticket)), + ) + files.append(file_data) + + logger.info(f"Zendesk tickets retrieved successfully: {len(files)}") + return files + + async def aget_files_by_id( + self, credentials: Dict, file_ids: List[str] + ) -> List[SyncFile]: + return self.get_files_by_id(credentials, file_ids) + + def download_file( + self, credentials: Dict, file: SyncFile + ) -> Dict[str, Union[str, BytesIO]]: + """ + Download a specific Zendesk ticket as a JSON file. + + Args: + credentials (Dict): Zendesk API credentials. + file_id (str): ID of the ticket to download. + file_name (str): Name of the file to be downloaded. + + Returns: + BytesIO: A BytesIO object containing the JSON data of the ticket. + """ + logger.info(f"Downloading Zendesk ticket with ID: {file.id}") + url = f"{self.get_zendesk_api_url(credentials)}/tickets/{file.id}/comments.json" + headers = { + "Content-Type": "application/json", + } + email_address = f"{credentials['email']}/token" + api_token = credentials["api_token"] + auth = (email_address, api_token) + + response = requests.get(url, auth=auth, headers=headers) + + ticket_data = response.json().get("comments", {}) + + # Convert the ticket data to a JSON string + json_data = json.dumps(ticket_data, indent=2) + + # Create a BytesIO object from the JSON string + file_content = BytesIO(json_data.encode("utf-8")) + + return { + "file_name": f"{file.name.rsplit('.', 1)[0]}.txt", + "content": file_content, + } + + async def adownload_file( + self, credentials: Dict, file: SyncFile + ) -> Dict[str, Union[str, BytesIO]]: + return self.download_file(credentials, file) + + def check_and_refresh_access_token(self, credentials: Dict) -> Dict: + # not needed for Zendesk + return credentials diff --git a/backend/api/quivr_api/modules/sync/utils/syncutils.py b/backend/api/quivr_api/modules/sync/utils/syncutils.py index 5fe9f53105b0..7594e1bf7f4b 100644 --- a/backend/api/quivr_api/modules/sync/utils/syncutils.py +++ b/backend/api/quivr_api/modules/sync/utils/syncutils.py @@ -132,7 +132,7 @@ async def download_file( ) extension = os.path.splitext(file_name)[-1].lower() dfile = DownloadedSyncFile( - file_name=file_name, + file_name=file_name.encode("ascii", errors="ignore").decode("ascii"), file_data=file_data, extension=extension, ) @@ -165,6 +165,7 @@ async def process_sync_file( ".xlsx", ".pptx", ".doc", + ".json", ]: raise ValueError(f"Incompatible file extension for {downloaded_file}") diff --git a/backend/requirements.lock b/backend/requirements.lock index 7bb40f61eaef..e2761ec741f3 100644 --- a/backend/requirements.lock +++ b/backend/requirements.lock @@ -611,6 +611,34 @@ numpy==1.26.3 # via torchvision # via transformers # via unstructured +nvidia-cublas-cu12==12.1.3.1 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via nvidia-cudnn-cu12 + # via nvidia-cusolver-cu12 + # via torch +nvidia-cuda-cupti-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cuda-nvrtc-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cuda-runtime-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cudnn-cu12==9.1.0.70 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cufft-cu12==11.0.2.54 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-curand-cu12==10.3.2.106 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cusolver-cu12==11.4.5.107 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-cusparse-cu12==12.1.0.106 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via nvidia-cusolver-cu12 + # via torch +nvidia-nccl-cu12==2.20.5 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch +nvidia-nvjitlink-cu12==12.6.77 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via nvidia-cusolver-cu12 + # via nvidia-cusparse-cu12 +nvidia-nvtx-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' + # via torch oauthlib==3.2.2 # via requests-oauthlib olefile==0.47 @@ -1032,23 +1060,13 @@ tokenizers==0.19.1 # via cohere # via litellm # via transformers -torch==2.4.0 ; platform_machine != 'x86_64' +torch==2.4.0 # via effdet # via quivr-worker # via timm # via torchvision # via unstructured-inference -torch==2.4.0+cpu ; platform_machine == 'x86_64' - # via effdet - # via quivr-worker - # via timm - # via torchvision - # via unstructured-inference -torchvision==0.19.0 ; platform_machine != 'x86_64' - # via effdet - # via quivr-worker - # via timm -torchvision==0.19.0+cpu ; platform_machine == 'x86_64' +torchvision==0.19.0 # via effdet # via quivr-worker # via timm diff --git a/backend/worker/quivr_worker/syncs/utils.py b/backend/worker/quivr_worker/syncs/utils.py index bbc3c75f8588..7911af61d9b9 100644 --- a/backend/worker/quivr_worker/syncs/utils.py +++ b/backend/worker/quivr_worker/syncs/utils.py @@ -23,6 +23,7 @@ GitHubSync, GoogleDriveSync, NotionSync, + ZendeskSync, ) from quivr_api.modules.sync.utils.syncutils import SyncUtils from sqlalchemy.ext.asyncio import AsyncEngine @@ -70,6 +71,7 @@ async def build_syncs_utils( "notion", NotionSync(notion_service=notion_service), ), # Fixed duplicate "github" key + ("zendesk", ZendeskSync()), ]: provider_sync_util = SyncUtils( sync_user_service=deps.sync_user_service, diff --git a/frontend/lib/api/sync/sync.ts b/frontend/lib/api/sync/sync.ts index c2a16004772e..f91f88a29421 100644 --- a/frontend/lib/api/sync/sync.ts +++ b/frontend/lib/api/sync/sync.ts @@ -37,6 +37,17 @@ export const syncSharepoint = async ( ).data; }; +export const syncZendesk = async ( + name: string, + axiosInstance: AxiosInstance +): Promise<{ authorization_url: string }> => { + return ( + await axiosInstance.post<{ authorization_url: string }>( + `/sync/zendesk/authorize?name=${name}` + ) + ).data; +}; + export const syncDropbox = async ( name: string, axiosInstance: AxiosInstance diff --git a/frontend/lib/api/sync/types.ts b/frontend/lib/api/sync/types.ts index 75fae2764adc..4ac1daba3fe0 100644 --- a/frontend/lib/api/sync/types.ts +++ b/frontend/lib/api/sync/types.ts @@ -1,11 +1,12 @@ -export type Provider = "Google" | "Azure" | "DropBox" | "Notion" | "GitHub"; +export type Provider = "Google" | "Azure" | "DropBox" | "Notion" | "GitHub" | "Zendesk"; export type Integration = | "Google Drive" | "Share Point" | "Dropbox" | "Notion" - | "GitHub"; + | "GitHub" + | "Zendesk" export type SyncStatus = "SYNCING" | "SYNCED" | "ERROR" | "REMOVED"; diff --git a/frontend/lib/api/sync/useSync.ts b/frontend/lib/api/sync/useSync.ts index 141ce71c97d3..ff187a776066 100644 --- a/frontend/lib/api/sync/useSync.ts +++ b/frontend/lib/api/sync/useSync.ts @@ -13,7 +13,8 @@ import { syncGoogleDrive, syncNotion, syncSharepoint, - updateActiveSync + syncZendesk, + updateActiveSync, } from "./sync"; import { Integration, OpenedConnection, Provider } from "./types"; @@ -32,6 +33,8 @@ export const useSync = () => { "https://quivr-cms.s3.eu-west-3.amazonaws.com/Notion_app_logo_004168672c.png", GitHub: "https://quivr-cms.s3.eu-west-3.amazonaws.com/dropbox_dce4f3d753.png", + Zendesk: + "https://quivr-cms.s3.eu-west-3.amazonaws.com/zendesk_c39745607c.png", }; const integrationIconUrls: Record = { @@ -45,6 +48,8 @@ export const useSync = () => { "https://quivr-cms.s3.eu-west-3.amazonaws.com/Notion_app_logo_004168672c.png", GitHub: "https://quivr-cms.s3.eu-west-3.amazonaws.com/dropbox_dce4f3d753.png", + Zendesk: + "https://quivr-cms.s3.eu-west-3.amazonaws.com/zendesk_c39745607c.png", }; const getActiveSyncsForBrain = async (brainId: string) => { @@ -59,6 +64,7 @@ export const useSync = () => { syncSharepoint: async (name: string) => syncSharepoint(name, axiosInstance), syncDropbox: async (name: string) => syncDropbox(name, axiosInstance), syncNotion: async (name: string) => syncNotion(name, axiosInstance), + syncZendesk: async (name: string) => syncZendesk(name, axiosInstance), getUserSyncs: async () => getUserSyncs(axiosInstance), getSyncFiles: async (userSyncId: number, folderId?: string) => getSyncFiles(axiosInstance, userSyncId, folderId), diff --git a/frontend/lib/components/ConnectionCards/ConnectionCards.tsx b/frontend/lib/components/ConnectionCards/ConnectionCards.tsx index ae81175a8302..a7c2c62ae1eb 100644 --- a/frontend/lib/components/ConnectionCards/ConnectionCards.tsx +++ b/frontend/lib/components/ConnectionCards/ConnectionCards.tsx @@ -10,7 +10,7 @@ interface ConnectionCardsProps { export const ConnectionCards = ({ fromAddKnowledge, }: ConnectionCardsProps): JSX.Element => { - const { syncGoogleDrive, syncSharepoint, syncDropbox } = + const { syncGoogleDrive, syncSharepoint, syncDropbox, syncZendesk } = useSync(); return ( @@ -37,6 +37,12 @@ export const ConnectionCards = ({ fromAddKnowledge={fromAddKnowledge} oneAccountLimitation={true} /> */} + syncZendesk(name)} + fromAddKnowledge={fromAddKnowledge} + />