backend: remove file paths from /chat (#652)

* wip

* fix tests

* remove file path from factory

* fix tests

Author: tianjing-li

Makefile

@@ -44,7 +44,7 @@ exec-db:
 
 .PHONY: migration
 migration:
-	docker compose run --build backend alembic -c src/backend/alembic.ini revision --autogenerate
+	docker compose run --build backend alembic -c src/backend/alembic.ini revision --autogenerate -m "$(message)"
 
 .PHONY: migrate
 migrate:

src/backend/alembic/versions/2024_08_21_08bcb9a24d9b_remove_file_path.py

@@ -0,0 +1,29 @@
+"""Remove file path
+
+Revision ID: 08bcb9a24d9b
+Revises: c301506b3676
+Create Date: 2024-08-21 15:59:18.678457
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = '08bcb9a24d9b'
+down_revision: Union[str, None] = 'c301506b3676'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('files', 'file_path')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('files', sa.Column('file_path', sa.VARCHAR(), autoincrement=False, nullable=False))
+    # ### end Alembic commands ###

src/backend/database_models/file.py

@@ -9,7 +9,6 @@ class File(Base):
 
     user_id: Mapped[str] = mapped_column(String, nullable=True)
     file_name: Mapped[str]
-    file_path: Mapped[str]
     file_size: Mapped[int] = mapped_column(default=0)
     file_content: Mapped[str] = mapped_column(default="")
 

src/backend/schemas/file.py

@@ -12,7 +12,6 @@ class File(BaseModel):
     conversation_id: str
     file_content: str
     file_name: str
-    file_path: str
     file_size: int = Field(default=0, ge=0)
 
     class Config:
@@ -26,7 +25,6 @@ class ConversationFilePublic(BaseModel):
 
     conversation_id: str
     file_name: str
-    file_path: str
     file_size: int = Field(default=0, ge=0)
 
 
@@ -37,7 +35,6 @@ class AgentFilePublic(BaseModel):
     updated_at: datetime.datetime
 
     file_name: str
-    file_path: str
     file_size: int = Field(default=0, ge=0)
 
 class ListConversationFile(ConversationFilePublic):

src/backend/services/chat.py

@@ -44,7 +44,6 @@
     StreamToolResult,
     ToolInputType,
 )
-from backend.schemas.cohere_chat import CohereChatRequest
 from backend.schemas.context import Context
 from backend.schemas.conversation import UpdateConversationRequest
 from backend.schemas.search_query import SearchQuery
@@ -127,14 +126,13 @@ def process_chat(
         id=str(uuid4()),
     )
 
-    if isinstance(chat_request, CohereChatRequest):
-        if should_store:
-            attach_files_to_messages(
-                session,
-                user_id,
-                user_message.id,
-                chat_request.file_ids,
-            )
+    if should_store:
+        attach_files_to_messages(
+            session,
+            user_id,
+            user_message.id,
+            chat_request.file_ids
+        )
 
     chat_history = create_chat_history(
         conversation, next_message_position, chat_request

src/backend/services/file.py

@@ -430,7 +430,6 @@ def get_files_in_compass(
                 id=file_id,
                 file_name=fetched_doc["file_name"],
                 file_size=fetched_doc["file_size"],
-                file_path=fetched_doc["file_path"],
                 file_content=fetched_doc["text"],
                 user_id=user_id,
                 created_at=datetime.fromisoformat(fetched_doc["created_at"]),
@@ -491,7 +490,6 @@ async def consolidate_agent_files_in_compass(
                     "custom_context": {
                         "file_id": file_id,
                         "file_name": fetched_doc["file_name"],
-                        "file_path": fetched_doc["file_path"],
                         "file_size": fetched_doc["file_size"],
                         "user_id": fetched_doc["user_id"],
                         "created_at": fetched_doc["created_at"],
@@ -574,7 +572,6 @@ async def insert_files_in_compass(
                     "custom_context": {
                         "file_id": new_file_id,
                         "file_name": filename,
-                        "file_path": filename,
                         "file_size": file.size,
                         "user_id": user_id,
                         "created_at": datetime.now().isoformat(),
@@ -596,7 +593,6 @@ async def insert_files_in_compass(
                 file_name=filename,
                 id=new_file_id,
                 file_size=file.size,
-                file_path=filename,
                 user_id=user_id,
                 created_at=datetime.now(),
                 updated_at=datetime.now(),
@@ -661,7 +657,6 @@ async def insert_files_in_db(
             FileModel(
                 file_name=filename,
                 file_size=file.size,
-                file_path=filename,
                 file_content=cleaned_content,
                 user_id=user_id,
             )
@@ -682,7 +677,6 @@ def attach_conversation_id_to_files(
                 conversation_id=conversation_id,
                 file_name=file.file_name,
                 file_size=file.file_size,
-                file_path=file.file_path,
                 user_id=file.user_id,
                 created_at=file.created_at,
                 updated_at=file.updated_at,

src/backend/tests/unit/crud/test_file.py

@@ -13,34 +13,29 @@ def conversation(session, user):
 def test_create_file(session, user):
     file_data = File(
         file_name="test.txt",
-        file_path="/tmp/test.txt",
         file_size=100,
         user_id=user.id,
     )
 
     file = file_crud.create_file(session, file_data)
     assert file.file_name == file_data.file_name
-    assert file.file_path == file_data.file_path
     assert file.file_size == file_data.file_size
     assert file.user_id == file_data.user_id
 
     file = file_crud.get_file(session, file.id, user.id)
     assert file.file_name == file_data.file_name
-    assert file.file_path == file_data.file_path
     assert file.file_size == file_data.file_size
     assert file.user_id == file_data.user_id
 
 
 def test_batch_create_files(session, user):
     file_data = File(
         file_name="test.txt",
-        file_path="/tmp/test.txt",
         file_size=100,
         user_id=user.id,
     )
     file_data2 = File(
         file_name="test2.txt",
-        file_path="/tmp/test2.txt",
         file_size=100,
         user_id=user.id,
     )

src/backend/tests/unit/factories/file.py

@@ -11,5 +11,4 @@ class Meta:
 
     user_id = factory.Faker("uuid4")
     file_name = factory.Faker("file_name")
-    file_path = factory.Faker("file_path")
     file_size = factory.Faker("random_int", min=1, max=20000000)

src/backend/tools/tavily.py

@@ -140,7 +140,7 @@ def to_langchain_tool(self) -> TavilySearchResults:
         internet_search.name = "internet_search"
         internet_search.description = "Returns a list of relevant document snippets for a textual query retrieved from the internet."
 
-        # pydantic v1 base model
+        # Pydantic v1 base model
         from langchain_core.pydantic_v1 import BaseModel, Field
 
         class TavilySearchInput(BaseModel):