wip

Author: leoguillaume

.github/compose.test.yml

@@ -0,0 +1,43 @@
+name: albert-api
+services:
+  api:
+    build:
+      context: ..
+      dockerfile: app/Dockerfile
+    platform: linux/amd64
+    restart: always
+    environment:
+      - COVERAGE_RCFILE=./app/.coveragerc
+      - BRAVE_API_KEY=${BRAVE_API_KEY}
+      - ALBERT_API_KEY=${ALBERT_API_KEY}
+    ports:
+      - 8000:8000
+    volumes:
+      - ./config.test.yml:/config.yml:ro
+    depends_on:
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      qdrant:
+        condition: service_healthy
+
+  postgres:
+    extends:
+      file: ../compose.yml
+      service: postgres
+
+  redis:
+    extends:
+      file: ../compose.yml
+      service: redis
+
+  qdrant:
+    extends:
+      file: ../compose.yml
+      service: qdrant
+
+volumes:
+  postgres:
+  redis:
+  qdrant:

.github/config.test.yml

@@ -0,0 +1,100 @@
+general:
+  log_level: DEBUG
+
+playground:
+  api_url: http://api:8000
+  max_api_key_expiration_days: 365
+  cache_ttl: 1800
+  database_url: postgresql://postgres:changeme@postgres:5432/playground
+
+auth:
+  master_username: master
+  master_key: changeme
+
+web_search:
+  - type: brave
+    model: albert-small
+    args:
+      api_key: ${BRAVE_API_KEY}
+
+databases:
+  - type: qdrant
+    model: embeddings-small
+    args:
+      url: http://qdrant
+      api_key: changeme
+      prefer_grpc: True
+      timeout: 10
+
+  - type: redis
+    args:
+      host: redis
+      password: changeme
+
+  - type: sql
+    args:
+      url: postgresql+asyncpg://postgres:changeme@postgres:5432/api
+      echo: True
+      pool_size: 5
+      max_overflow: 10
+      pool_pre_ping: True
+      connect_args: {"server_settings": {"statement_timeout": "120s"}}
+
+models:
+  - id: albert-large
+    type: text-generation
+    owned_by: test
+    aliases: ["mistralai/Mistral-Small-3.1-24B-Instruct-2503"]
+    clients:
+      - model: mistralai/Mistral-Small-3.1-24B-Instruct-2503
+        type: albert
+        args:
+          api_url: https://albert.api.etalab.gouv.fr
+          api_key: ${ALBERT_API_KEY}
+          timeout: 120
+
+  - id: albert-small
+    type: text-generation
+    aliases: ["meta-llama/Llama-3.1-8B-Instruct"]
+    clients:
+      - model: albert-small
+        type: albert
+        args:
+          api_url: https://albert.api.etalab.gouv.fr
+          api_key: ${ALBERT_API_KEY}
+          timeout: 120
+
+  - id: embeddings-small
+    type: text-embeddings-inference
+    aliases: ["BAAI/bge-m3"]
+    clients:
+      - model: BAAI/bge-m3
+        type: albert
+        args:
+          api_url: https://albert.api.etalab.gouv.fr
+          api_key: ${ALBERT_API_KEY}
+          timeout: 120
+
+  - id: audio-large
+    type: automatic-speech-recognition
+    aliases: ["openai/whisper-large-v3"]
+    clients: 
+      - model: audio-large
+        type: albert
+        args:
+          api_url: https://albert.api.etalab.gouv.fr
+          api_key: ${ALBERT_API_KEY}
+          timeout: 120
+
+  - id: rerank-small
+    type: text-classification
+    aliases: ["BAAI/bge-reranker-v2-m3"]
+    clients:
+      - model: rerank-small
+        type: albert
+        args:
+          api_url: https://albert.api.etalab.gouv.fr
+          api_key: ${ALBERT_API_KEY}
+          timeout: 120
+         
+

.github/workflows/run_tests.yml

@@ -13,20 +13,22 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-
-    - name: Create config from template
-      run: |
-        echo "${{ secrets.CONFIG_YML }}" > config.yml
 
     - name: Set up Docker Compose
       run: |
-        docker compose --file compose.dev.yml up --detach
+        docker compose --file ./.github/compose.test.yml up --detach
+      env:
+        BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
+        ALBERT_API_KEY: ${{ secrets.ALBERT_API_KEY }}
 
     - name: Wait for API to start
       run: |
-        sleep 30
+        echo $(ls -la)
+        for i in {1..30}; do
+          curl -s http://localhost:8000/health -H "Authorization: Bearer changeme" > /dev/null && echo "API is ready" && break || echo "Waiting for API..." && sleep 2;
+        done
         echo $(docker logs albert-api-api-1)
-
+  
     - name: Wait for PostgreSQL
       run: |
         for i in {1..30}; do

app/endpoints/documents.py

@@ -46,15 +46,16 @@ async def get_documents(
     if not context.documents:  # no vector store available
         if collection:
             raise CollectionNotFoundException()
-        data = []
-    else:
-        data = await context.documents.get_documents(
-            session=session,
-            collection_id=collection,
-            limit=limit,
-            offset=offset,
-            user_id=request.app.state.user.id,
-        )
+
+        return Documents(data=[])
+
+    data = await context.documents.get_documents(
+        session=session,
+        collection_id=collection,
+        limit=limit,
+        offset=offset,
+        user_id=request.app.state.user.id,
+    )
 
     return Documents(data=data)
 

app/helpers/_documentmanager.py

@@ -19,7 +19,7 @@
     VectorParams,
 )
 from sqlalchemy import Integer, cast, delete, distinct, func, insert, or_, select, update
-from sqlalchemy.exc import IntegrityError, NoResultFound
+from sqlalchemy.exc import NoResultFound
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.clients.model import BaseModelClient as ModelClient
@@ -36,7 +36,6 @@
 from app.sql.models import User as UserTable
 from app.utils.exceptions import (
     ChunkingFailedException,
-    CollectionAlreadyExistsException,
     CollectionNotFoundException,
     DocumentNotFoundException,
     NotImplementedException,
@@ -75,16 +74,13 @@ def __init__(
     async def create_collection(
         self, session: AsyncSession, user_id: int, name: str, visibility: CollectionVisibility, description: Optional[str] = None
     ) -> int:
-        try:
-            result = await session.execute(
-                statement=insert(table=CollectionTable)
-                .values(name=name, user_id=user_id, visibility=visibility, description=description)
-                .returning(CollectionTable.id)
-            )
-            collection_id = result.scalar_one()
-            await session.commit()
-        except IntegrityError as e:
-            raise CollectionAlreadyExistsException()
+        result = await session.execute(
+            statement=insert(table=CollectionTable)
+            .values(name=name, user_id=user_id, visibility=visibility, description=description)
+            .returning(CollectionTable.id)
+        )
+        collection_id = result.scalar_one()
+        await session.commit()
 
         await self.qdrant.create_collection(
             collection_name=str(collection_id),

app/helpers/_identityaccessmanager.py

@@ -16,7 +16,6 @@
     DeleteRoleWithUsersException,
     RoleAlreadyExistsException,
     RoleNotFoundException,
-    TokenAlreadyExistsException,
     TokenNotFoundException,
     UserAlreadyExistsException,
     UserNotFoundException,
@@ -335,12 +334,9 @@ async def create_token(self, session: AsyncSession, user_id: int, name: str, exp
             raise UserNotFoundException()
 
         # create the token
-        try:
-            result = await session.execute(statement=insert(table=TokenTable).values(user_id=user.id, name=name).returning(TokenTable.id))
-            token_id = result.scalar_one()
-            await session.commit()
-        except IntegrityError:
-            raise TokenAlreadyExistsException()
+        result = await session.execute(statement=insert(table=TokenTable).values(user_id=user.id, name=name).returning(TokenTable.id))
+        token_id = result.scalar_one()
+        await session.commit()
 
         # generate the token
         token = self._encode_token(user_id=user.id, token_id=token_id, expires_at=expires_at)

app/main.py

@@ -41,7 +41,7 @@ def create_app(db_func=get_db, *args, **kwargs) -> FastAPI:
     )
 
     # Middlewares
-    if not settings.general.disabled_middleware:
+    if not settings.general.disabled_middlewares:
         app.add_middleware(middleware_class=UsagesMiddleware, db_func=get_db)
         app.instrumentator = Instrumentator().instrument(app=app)
 
@@ -77,7 +77,7 @@ def create_app(db_func=get_db, *args, **kwargs) -> FastAPI:
         app.include_router(router=models.router, tags=[ROUTER__MODELS.title()], prefix="/v1")
 
     if ROUTER__MONITORING not in settings.general.disabled_routers:
-        if not settings.general.disabled_middleware:
+        if not settings.general.disabled_middlewares:
             app.instrumentator.expose(app=app, should_gzip=True, tags=[ROUTER__MONITORING.title()], dependencies=[Depends(dependency=Authorization(permissions=[PermissionType.READ_METRIC]))], include_in_schema=settings.general.log_level == "DEBUG")  # fmt: off
 
         @app.get(path="/health", tags=[ROUTER__MONITORING.title()], include_in_schema=settings.general.log_level == "DEBUG", dependencies=[Security(dependency=Authorization())])  # fmt: off

app/schemas/core/settings.py

@@ -125,7 +125,7 @@ class General(ConfigBaseModel):
 
     # Others
     disabled_routers: List[Literal[*ROUTERS]] = []
-    disabled_middleware: bool = False
+    disabled_middlewares: bool = False
     log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
 
 
@@ -175,7 +175,7 @@ def setup_config(cls, values) -> Any:
         # replace environment variables (pattern: ${VARIABLE_NAME})
         for match in set(re.findall(pattern=r"\${[A-Z_]+}", string=file_content)):
             variable = match.replace("${", "").replace("}", "")
-            if not os.getenv(variable):
+            if os.getenv(variable) is None or os.getenv(variable) == "":
                 logging.warning(f"Environment variable {variable} not found or empty to replace {match}.")
             file_content = file_content.replace(match, os.getenv(variable, match))
 

app/tests/conftest.py

@@ -96,7 +96,7 @@ def get_test_db():
         yield db_session
 
     # Create app with test config
-    app = create_app(db_func=get_test_db, disabled_middleware=False)
+    app = create_app(db_func=get_test_db)
 
     # Exit the global cassette, requests done by app initialization
     # are recorded in the global cassette

app/utils/exceptions.py

@@ -19,11 +19,6 @@ def __init__(self, detail: str = "Web search is not available."):
         super().__init__(status_code=400, detail=detail)
 
 
-class CollectionAlreadyExistsException(HTTPException):
-    def __init__(self, detail: str = "Collection already exists."):
-        super().__init__(status_code=400, detail=detail)
-
-
 class RoleAlreadyExistsException(HTTPException):
     def __init__(self, detail: str = "Role already exists."):
         super().__init__(status_code=400, detail=detail)
@@ -39,11 +34,6 @@ def __init__(self, detail: str = "User already exists."):
         super().__init__(status_code=400, detail=detail)
 
 
-class TokenAlreadyExistsException(HTTPException):
-    def __init__(self, detail: str = "Token already exists."):
-        super().__init__(status_code=400, detail=detail)
-
-
 # 403
 class InvalidPasswordException(HTTPException):
     def __init__(self, detail: str = "Invalid password."):