feat: qdrant grist migration

Author: leoguillaume

README.md

@@ -1,7 +1,7 @@
 <div id="toc"><ul align="center" style="list-style: none">
 <summary><h1>Albert API</h1></summary>
 
-![Code Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/etalab-ia/albert-api/174-tests-CI-CD/.github/badges/coverage.json)
+![codecov](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/etalab-ia/albert-api/174-tests-CI-CD/.github/badges/coverage.json)
 
 <br>
 <a href="https://github.com/etalab-ia/albert-api/blob/main/CHANGELOG.md"><b>Changelog</b></a> | <a href="https://albert.api.etalab.gouv.fr/documentation"><b>Documentation</b></a> | <a href="https://albert.api.etalab.gouv.fr/playground"><b>Playground</b></a> | <a href="https://albert.api.etalab.gouv.fr/status"><b>Status</b></a> | <a href="https://albert.api.etalab.gouv.fr/swagger"><b>Swagger</b></a> <br><br>
@@ -18,10 +18,6 @@ Ce framework, destiné à un environnement de production soumis à des contraint
 
 En se basant sur les conventions définies par OpenAI, Albert API expose des endpoints qui peuvent être appelés avec le [client officiel python d'OpenAI](https://github.com/openai/openai-python/tree/main). Ce formalisme permet une intégration aisée avec des bibliothèques tierces comme [Langchain](https://www.langchain.com/) ou [LlamaIndex](https://www.llamaindex.ai/).
 
-## 🔑 Accès
-
-Si vous êtes un organisme public, vous pouvez demander une clé d'accès à Albert API en remplissant le [formulaire sur le site ALLiaNCE](https://alliance.numerique.gouv.fr/albert/).
-
 ## 📫 API Gateway
 
 L'API Albert permet d'être un proxy entre des clients API d'IA générative et d'assurer du load balancing entre les différents clients :
@@ -101,22 +97,10 @@ Vous trouverez ici des ressources de documentation :
 - [Documentation technique de l'API](./docs)
 - [Repository HuggingFace](https://huggingface.co/AgentPublic)
 
-## 🧑‍💻 Contribuez au projet
-
-Albert API est un projet open source, vous pouvez contribuer au projet en lisant notre [guide de contribution](./CONTRIBUTING.md).
-
-## 🚀 Installation
+## 🚀 Quickstart
 
 Pour déployer l'API Albert sur votre propre infrastructure, suivez la [documentation](./docs/deployment.md).
 
-### Quickstart
-
-1. Complétez le fichier *[config.example.yml](./config.example.yml)* à la racine du dépot la configuration de vos API de modèles (voir la [documentation déploiement](./docs/deployment.md) pour plus d'informations).
-
-2. Complétez le fichier *[compose.yml](./compose.yml)* à la racine du dépot avec les variables d'environnement nécessaires pour l'UI.
-
-3. Déployez l'API avec Docker à l'aide du fichier [compose.yml](../compose.yml) à la racine du dépot.
+## 🧑‍💻 Contribuez au projet
 
-  ```bash
-  docker compose --file compose.prod.yml up --detach
-  ```
+Albert API est un projet open source, vous pouvez contribuer au projet en lisant notre [guide de contribution](./CONTRIBUTING.md).

app/alembic/versions/2025_04_15_1513-a4ac45e7c990_remove_unique_name_on_token_and_.py

@@ -0,0 +1,34 @@
+"""remove unique name on token and collection
+
+Revision ID: a4ac45e7c990
+Revises: 8c5ae2a3d4d0
+Create Date: 2025-04-15 15:13:33.886841
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "a4ac45e7c990"
+down_revision: Union[str, None] = "8c5ae2a3d4d0"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint("unique_collection_name_per_user", "collection", type_="unique")
+    op.drop_constraint("unique_token_name_per_user", "token", type_="unique")
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_unique_constraint("unique_token_name_per_user", "token", ["user_id", "name"])
+    op.create_unique_constraint("unique_collection_name_per_user", "collection", ["user_id", "name"])
+    # ### end Alembic commands ###

app/endpoints/collections.py

@@ -1,17 +1,16 @@
 from typing import Union
 
-from fastapi import APIRouter, Depends, Path, Query, Request, Response, Security
+from fastapi import APIRouter, Body, Depends, Path, Query, Request, Response, Security
 from fastapi.responses import JSONResponse
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.helpers import Authorization
-from app.schemas.collections import Collection, CollectionRequest, CollectionUpdateRequest, Collections
+from app.schemas.collections import Collection, CollectionRequest, Collections, CollectionUpdateRequest
 from app.sql.session import get_db as get_session
 from app.utils.exceptions import CollectionNotFoundException
 from app.utils.lifespan import context
 from app.utils.variables import ENDPOINT__COLLECTIONS
 
-
 router = APIRouter()
 
 
@@ -101,7 +100,7 @@ async def delete_collections(
 async def update_collection(
     request: Request,
     collection: int = Path(..., description="The collection ID"),
-    body: CollectionUpdateRequest = None,
+    body: CollectionUpdateRequest = Body(..., description="The collection to update."),
     session: AsyncSession = Depends(get_session),
 ) -> Response:
     """
@@ -114,9 +113,9 @@ async def update_collection(
         session=session,
         user_id=request.app.state.user.id,
         collection_id=collection,
-        name=body.name if body else None,
-        visibility=body.visibility if body else None,
-        description=body.description if body else None,
+        name=body.name,
+        visibility=body.visibility,
+        description=body.description,
     )
 
     return Response(status_code=204)

app/helpers/_authorization.py

@@ -57,6 +57,9 @@ async def __call__(
         if request.url.path.startswith(f"/v1{ENDPOINT__CHAT_COMPLETIONS}") and request.method == "POST":
             await self._check_chat_completions_post(user=user, role=role, limits=limits, request=request)
 
+        if request.url.path.startswith(f"/v1{ENDPOINT__COLLECTIONS}") and request.method == "PATCH":
+            await self._check_collections_patch(user=user, role=role, limits=limits, request=request)
+
         if request.url.path.startswith(f"/v1{ENDPOINT__COLLECTIONS}") and request.method == "POST":
             await self._check_collections_post(user=user, role=role, limits=limits, request=request)
 
@@ -180,6 +183,13 @@ async def _check_chat_completions_post(self, user: User, role: Role, limits: Dic
         if body.get("search", False):
             await self._check_limits(user=user, limits=limits, model=body.get("search_args", {}).get("model", None))
 
+    async def _check_collections_patch(self, user: User, role: Role, limits: Dict[str, UserModelLimits], request: Request) -> None:
+        body = await request.body()
+        body = json.loads(body)
+
+        if body.get("visibility") == CollectionVisibility.PUBLIC and PermissionType.CREATE_PUBLIC_COLLECTION not in role.permissions:
+            raise InsufficientPermissionException("Missing permission to update collection visibility to public.")
+
     async def _check_collections_post(self, user: User, role: Role, limits: Dict[str, UserModelLimits], request: Request) -> None:
         body = await request.body()
         body = json.loads(body)

app/helpers/_documentmanager.py

@@ -180,8 +180,8 @@ async def create_document(self, session: AsyncSession, user_id: int, collection_
         except NoResultFound:
             raise CollectionNotFoundException()
 
-        file_name = file.filename.strip()
-        file_extension = file_name.rsplit(".", maxsplit=1)[-1]
+        document_name = file.filename.strip()
+        file_extension = document_name.rsplit(".", maxsplit=1)[-1]
 
         try:
             document = self._parse(file=file, file_extension=file_extension)
@@ -198,19 +198,18 @@ async def create_document(self, session: AsyncSession, user_id: int, collection_
             raise ChunkingFailedException(detail=f"Chunking failed: {e}")
 
         result = await session.execute(
-            statement=insert(table=DocumentTable).values(name=file_name, collection_id=collection_id).returning(DocumentTable.id)
+            statement=insert(table=DocumentTable).values(name=document_name, collection_id=collection_id).returning(DocumentTable.id)
         )
         document_id = result.scalar_one()
         await session.commit()
 
         client = self.qdrant_model.get_client(endpoint=ENDPOINT__EMBEDDINGS)
         for i, chunk in enumerate(chunks):
-            chunk.metadata["document_part"] = f"{i + 1}/{len(chunks)}"
             chunk.metadata["collection_id"] = collection.id
-            chunk.metadata["collection_name"] = collection.name
             chunk.metadata["document_id"] = document_id
-            chunk.metadata["document_name"] = file_name
+            chunk.metadata["document_name"] = document_name
             chunk.metadata["document_created_at"] = round(time.time())
+            chunk.metadata["document_part"] = f"{i + 1}/{len(chunks)}"
         try:
             await self._upsert(chunks=chunks, collection_id=collection_id, model_client=client)
         except Exception as e:
@@ -279,7 +278,13 @@ async def delete_document(self, session: AsyncSession, user_id: int, document_id
         await self.qdrant.delete(collection_name=str(document.collection_id), points_selector=FilterSelector(filter=filter))
 
     async def get_chunks(
-        self, session: AsyncSession, user_id: int, document_id: int, chunk_id: Optional[int] = None, offset: int = 0, limit: int = 10
+        self,
+        session: AsyncSession,
+        user_id: int,
+        document_id: int,
+        chunk_id: Optional[int] = None,
+        offset: int = 0,
+        limit: int = 10,
     ) -> List[Chunk]:
         # check if document exists
         result = await session.execute(

app/sql/models.py

@@ -95,8 +95,6 @@ class Token(Base):
 
     user = relationship(argument="User", backref=backref(name="token", cascade="all, delete-orphan"))
 
-    __table_args__ = (UniqueConstraint("user_id", "name", name="unique_token_name_per_user"),)
-
 
 class Collection(Base):
     __tablename__ = "collection"
@@ -111,8 +109,6 @@ class Collection(Base):
 
     user = relationship(argument="User", backref=backref(name="collection", cascade="all, delete-orphan"))
 
-    __table_args__ = (UniqueConstraint("user_id", "name", name="unique_collection_name_per_user"),)
-
 
 class Document(Base):
     __tablename__ = "document"

app/tests/test_collections.py

@@ -10,7 +10,7 @@
 
 @pytest.mark.usefixtures("client")
 class TestCollections:
-    def test_create_private_collection_with_user(self, client: TestClient):
+    def test_create_private_collection(self, client: TestClient):
         params = {"name": f"test_collection_{str(uuid4())}", "visibility": CollectionVisibility.PRIVATE}
         response = client.post_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
         assert response.status_code == 201, response.text
@@ -30,7 +30,7 @@ def test_create_private_collection_with_user(self, client: TestClient):
         assert collection["name"] == params["name"]
         assert collection["visibility"] == CollectionVisibility.PRIVATE
 
-    def test_get_one_collection_with_user(self, client: TestClient):
+    def test_get_one_collection(self, client: TestClient):
         collection_name = f"test_collection_{str(uuid4())}"
         params = {"name": collection_name, "visibility": CollectionVisibility.PRIVATE}
         response = client.post_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
@@ -44,17 +44,6 @@ def test_get_one_collection_with_user(self, client: TestClient):
         collection = response.json()
         assert collection["name"] == collection_name
 
-    def test_create_private_collection_already_existing_name(self, client: TestClient):
-        collection_name = f"test_collection_{str(uuid4())}"
-        params = {"name": collection_name, "visibility": CollectionVisibility.PRIVATE}
-        response = client.post_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
-        assert response.status_code == 201, response.text
-
-        params = {"name": collection_name, "visibility": CollectionVisibility.PRIVATE}
-
-        response = client.post_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
-        assert response.status_code == 400, response.text
-
     def test_patch_collection_name(self, client: TestClient):
         collection_name = f"test_collection_{str(uuid4())}"
         params = {"name": collection_name, "visibility": CollectionVisibility.PRIVATE}
@@ -73,7 +62,7 @@ def test_patch_collection_name(self, client: TestClient):
         collection = response.json()
         assert collection["name"] == new_collection_name
 
-    def test_format_collection_with_user(self, client: TestClient):
+    def test_format_collection(self, client: TestClient):
         params = {"name": f"test_collection_{str(uuid4())}", "visibility": CollectionVisibility.PRIVATE}
         response = client.post_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
         assert response.status_code == 201, response.text
@@ -97,12 +86,17 @@ def test_format_collection_with_user(self, client: TestClient):
         collection = response.json()
         Collection(**collection)  # test output format
 
-    def test_create_public_collection_with_user(self, client: TestClient):
+    def test_create_public_collection_without_permissions(self, client: TestClient):
         params = {"name": f"test_collection_{str(uuid4())}", "visibility": CollectionVisibility.PUBLIC}
         response = client.post_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
         assert response.status_code == 403, response.text
 
-    def test_create_public_collection_with_admin(self, client: TestClient):
+    def test_patch_public_collection_without_permissions(self, client: TestClient):
+        params = {"name": f"test_collection_{str(uuid4())}", "visibility": CollectionVisibility.PUBLIC}
+        response = client.post_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
+        assert response.status_code == 403, response.text
+
+    def test_create_public_collection_with_permissions(self, client: TestClient):
         collection_name = f"test_collection_{str(uuid4())}"
         params = {"name": collection_name, "visibility": CollectionVisibility.PUBLIC}
         response = client.post_with_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
@@ -121,15 +115,17 @@ def test_create_public_collection_with_admin(self, client: TestClient):
         assert collection["name"] == collection_name
         assert collection["visibility"] == CollectionVisibility.PUBLIC
 
-    def test_create_already_existing_collection_with_user(self, client: TestClient):
+    def test_patch_public_collection_with_permissions(self, client: TestClient):
         collection_name = f"test_collection_{str(uuid4())}"
         params = {"name": collection_name, "visibility": CollectionVisibility.PRIVATE}
-        response = client.post_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
+        response = client.post_with_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
         assert response.status_code == 201, response.text
 
-        params = {"name": collection_name, "visibility": CollectionVisibility.PRIVATE}
-        response = client.post_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
-        assert response.status_code == 400, response.text
+        collection_id = response.json()["id"]
+
+        params = {"visibility": CollectionVisibility.PUBLIC}
+        response = client.patch_with_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}/{collection_id}", json=params)
+        assert response.status_code == 204, response.text
 
     def test_view_collection_of_other_user(self, client: TestClient):
         collection_name = f"test-collection_{str(uuid4())}"
@@ -139,12 +135,8 @@ def test_view_collection_of_other_user(self, client: TestClient):
 
         collection_id = response.json()["id"]
 
-        response = client.get_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}")
-        collections = response.json()
-        assert response.status_code == 200, response.text
-
-        collections = [collection["id"] for collection in collections["data"] if collection["id"] == collection_id]
-        assert len(collections) == 0
+        response = client.get_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}/{collection_id}")
+        assert response.status_code == 404, response.text
 
     def test_view_public_collection_of_other_user(self, client: TestClient):
         collection_name = f"test-collection_{str(uuid4())}"
@@ -154,19 +146,10 @@ def test_view_public_collection_of_other_user(self, client: TestClient):
 
         collection_id = response.json()["id"]
 
-        response = client.get_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}")
-        collections = response.json()
+        response = client.get_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}/{collection_id}")
         assert response.status_code == 200, response.text
 
-        collections = [collection for collection in collections["data"] if collection["id"] == collection_id]
-        assert len(collections) == 1
-
-        collection = collections[0]
-        assert collection["name"] == collection_name
-        assert collection["owner"] == "test-user-admin"
-        assert collection["visibility"] == CollectionVisibility.PUBLIC
-
-    def test_delete_private_collection_with_user(self, client: TestClient):
+    def test_delete_private_collection_without_permissions(self, client: TestClient):
         collection_name = f"test-collection_{str(uuid4())}"
         params = {"name": collection_name, "visibility": CollectionVisibility.PRIVATE}
         response = client.post_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
@@ -184,7 +167,7 @@ def test_delete_private_collection_with_user(self, client: TestClient):
         collections = [collection for collection in collections["data"] if collection["id"] == collection_id]
         assert len(collections) == 0
 
-    def test_delete_public_collection_with_user(self, client: TestClient):
+    def test_delete_public_collection_without_permissions(self, client: TestClient):
         collection_name = f"test-collection_{str(uuid4())}"
         params = {"name": collection_name, "visibility": CollectionVisibility.PUBLIC}
         response = client.post_with_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)
@@ -199,7 +182,7 @@ def test_delete_public_collection_with_user(self, client: TestClient):
         response = client.delete_without_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}/{collection_id}")
         assert response.status_code == 404, response.text
 
-    def test_delete_public_collection_with_admin(self, client: TestClient):
+    def test_delete_public_collection_with_permissions(self, client: TestClient):
         collection_name = f"test-collection_{str(uuid4())}"
         params = {"name": collection_name, "visibility": CollectionVisibility.PUBLIC}
         response = client.post_with_permissions(url=f"/v1{ENDPOINT__COLLECTIONS}", json=params)

docs/architecture.md

@@ -21,7 +21,7 @@ flowchart LR
 
     subgraph **app/clients**
     redisclient[Redis - ConnectionPool]
-    sqlclient[SQLDatabaseClient]
+    sqlclient[SQLAlchemy - AsyncSession]
     qdrantclient[Qrant - AsyncQdrantClient]
     internetclient[BraveInternetClient<br>DuckduckgoInternetClient]
     modelclient@{ shape: processes, label: "VllmModelClient<br>TeiModelClient<br>AlbertModelClient<br>OpenaiModelClient" }