chore: update dependencies (primarily for llama_index) (#683)

Author: zjx20

README.md

@@ -10,7 +10,7 @@ You should install Python 3.11 first.
 
 ```bash
 curl -sSL https://install.python-poetry.org | python3 -
-poetry self add poetry-plugin-shell	
+poetry self add poetry-plugin-shell
 ```
 
 * install dependencies
@@ -50,7 +50,7 @@ poetry shell
 ```
 
 ```
-PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python celery -A config.celery worker -l INFO --pool gevent
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python celery -A config.celery worker -l INFO --pool threads
 ```
 
 * run the frontend

aperag/auth/validator.py

@@ -27,7 +27,6 @@
 from aperag.auth import tv
 from aperag.utils.constant import KEY_USER_ID, KEY_WEBSOCKET_PROTOCOL
 from aperag.db.models import ApiKeyToken, ApiKeyStatus
-from ninja.compatibility.request import get_headers
 from django.core.cache import cache
 from asgiref.sync import sync_to_async
 
@@ -59,7 +58,7 @@ async def get_user_from_api_key(key):
         return None
     if api_key.status == ApiKeyStatus.DELETED:
         return None
-    
+
     cache.set(cache_key, api_key.user)
     return api_key.user
 
@@ -77,7 +76,7 @@ class GlobalHTTPAuth(HttpAuthBase):
     openapi_scheme: str = "bearer"
     header: str = "Authorization"
     async def __call__(self, request: HttpRequest) -> Optional[Any]:
-        headers = get_headers(request)
+        headers = request.headers
         auth_value = headers.get(self.header)
         if not auth_value:
             return None
@@ -89,7 +88,7 @@ async def __call__(self, request: HttpRequest) -> Optional[Any]:
             return None
         token = " ".join(parts[1:])
         return await self.authenticate(request, token, parts[0].lower())
-    
+
     async def authenticate(self, request, token, scheme):
         if scheme == self.openapi_scheme:
             request.META[KEY_USER_ID] = get_user_from_token(token)

aperag/db/ops.py

@@ -46,12 +46,12 @@
 
 
 class PagedQuery(BaseModel):
-    page_number: Optional[int]
-    page_size: Optional[int]
-    match_key: Optional[str]
-    match_value: Optional[str]
-    order_by: Optional[str]
-    order_desc: Optional[bool]
+    page_number: Optional[int] = None
+    page_size: Optional[int] = None
+    match_key: Optional[str] = None
+    match_value: Optional[str] = None
+    order_by: Optional[str] = None
+    order_desc: Optional[bool] = None
 
 
 class PagedResult(BaseModel):

aperag/pipeline/base_pipeline.py

@@ -30,19 +30,19 @@
 
 class Message(BaseModel):
     id: str
-    query: Optional[str]
-    timestamp: Optional[int]
-    response: Optional[str]
-    urls: Optional[List[str]]
-    references: Optional[List[Dict]]
-    collection_id: Optional[str]
-    embedding_model: Optional[str]
-    embedding_size: Optional[int]
-    embedding_score_threshold: Optional[float]
-    embedding_topk: Optional[int]
-    llm_model: Optional[str]
-    llm_prompt_template: Optional[str]
-    llm_context_window: Optional[int]
+    query: Optional[str] = None
+    timestamp: Optional[int] = None
+    response: Optional[str] = None
+    urls: Optional[List[str]] = None
+    references: Optional[List[Dict]] = None
+    collection_id: Optional[str] = None
+    embedding_model: Optional[str] = None
+    embedding_size: Optional[int] = None
+    embedding_score_threshold: Optional[float] = None
+    embedding_topk: Optional[int] = None
+    llm_model: Optional[str] = None
+    llm_prompt_template: Optional[str] = None
+    llm_context_window: Optional[int] = None
 
 
 KUBE_CHAT_DOC_QA_REFERENCES = "|KUBE_CHAT_DOC_QA_REFERENCES|"
@@ -124,7 +124,7 @@ async def generate_related_question(self, related_question_prompt):
                     if question:
                         related_questions.append(question)
         else:
-            related_questions = [] 
+            related_questions = []
             if content=='':
                 return related_questions
             questions = re.sub(r'\n+', '\n', content).split('\n')
@@ -137,7 +137,7 @@ async def generate_related_question(self, related_question_prompt):
                     question = match.group(1)
                 related_questions.append(question)
         return related_questions
-    
+
     @staticmethod
     async def new_human_message(message, message_id):
         return Message(

aperag/readers/base_readers.py

@@ -1,10 +1,10 @@
 from typing import Dict, Type
 
-from llama_index.readers.base import BaseReader
-from llama_index.readers.file.docs_reader import DocxReader, PDFReader
-from llama_index.readers.file.ipynb_reader import IPYNBReader
-from llama_index.readers.file.mbox_reader import MboxReader
-from llama_index.readers.file.tabular_reader import PandasCSVReader
+from llama_index.core.readers.base import BaseReader
+from llama_index.readers.file.docs.base import DocxReader, PDFReader
+from llama_index.readers.file.ipynb import IPYNBReader
+from llama_index.readers.file.mbox import MboxReader
+from llama_index.readers.file.tabular import PandasCSVReader
 
 from aperag.readers.compose_audio_reader import ComposeAudioReader
 from aperag.readers.compose_image_reader import ComposeImageReader

aperag/readers/compose_audio_reader.py

@@ -2,8 +2,8 @@
 from typing import Dict, List, Optional
 
 import requests
-from llama_index.readers.base import BaseReader
-from llama_index.schema import Document
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document
 
 import config.settings as settings
 

aperag/readers/compose_image_reader.py

@@ -6,8 +6,8 @@
 
 import PIL.Image
 import requests
-from llama_index.readers.base import BaseReader
-from llama_index.schema import Document, ImageDocument
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document, ImageDocument
 from PIL import Image
 
 import config.settings as settings

aperag/readers/compressed_file_reader.py

@@ -2,8 +2,8 @@
 from pathlib import Path
 from typing import Dict, List, Optional
 
-from llama_index.readers.base import BaseReader
-from llama_index.schema import Document
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document
 
 logger = logging.getLogger(__name__)
 class CompressedFileReader(BaseReader):

aperag/readers/doc_reader.py

@@ -5,9 +5,9 @@
 from pathlib import Path
 from typing import Dict, List, Optional
 
-from llama_index.readers.base import BaseReader
-from llama_index.readers.file.docs_reader import DocxReader
-from llama_index.schema import Document
+from llama_index.core.readers.base import BaseReader
+from llama_index.readers.file.docs import DocxReader
+from llama_index.core.schema import Document
 
 logger = logging.getLogger(__name__)
 

aperag/readers/docx_reader.py

@@ -7,8 +7,8 @@
 from docx.oxml import CT_P, CT_Tbl
 from docx.table import Table, _Cell
 from docx.text.paragraph import Paragraph
-from llama_index.readers.base import BaseReader
-from llama_index.schema import Document
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document
 
 from aperag.utils.utils import Stacks