Merge pull request #244 from linagora/revert-234-feat/file_quota

Revert "Feat/file quota"

Author: Ahmath-Gadji

.github/workflows/api_tests.yml

@@ -88,4 +88,4 @@ jobs:
         if: always()
         working-directory: .github/workflows/api_tests
         run: |
-          docker compose down -v
+          docker compose down -v

.github/workflows/api_tests/docker-compose.yaml

@@ -107,7 +107,6 @@ services:
       - RAY_ENABLE_UV_RUN_RUNTIME_ENV=0
       - RAY_memory_monitor_refresh_ms=0
       - PROMPTS_DIR=../prompts/example1
-      - DEFAULT_FILE_QUOTA=10
       - AUTH_TOKEN=${AUTH_TOKEN:-test-admin-token}
       - SUPER_ADMIN_MODE=${SUPER_ADMIN_MODE:-true}
     ports:
@@ -129,4 +128,4 @@ services:
       start_period: 60s
 
 volumes:
-  test_data:
+  test_data:

.gitmodules

@@ -4,4 +4,3 @@
 [submodule "extern/indexer-ui"]
 	path = extern/indexer-ui
 	url = https://github.com/linagora/openrag-admin-ui.git
-	branch = main

.hydra_config/config.yaml

@@ -46,7 +46,6 @@ rdb:
   port: ${oc.env:POSTGRES_PORT, 5432}
   user: ${oc.env:POSTGRES_USER, root}
   password: ${oc.env:POSTGRES_PASSWORD, root_password}
-  default_file_quota: ${oc.decode:${oc.env:DEFAULT_FILE_QUOTA, -1}}
 
 reranker:
   enable: ${oc.decode:${oc.env:RERANKER_ENABLED, true}}
@@ -172,4 +171,4 @@ ray:
     num_replicas: ${oc.decode:${oc.env:RAY_SERVE_NUM_REPLICAS, 1}}
     host: ${oc.env:RAY_SERVE_HOST, 0.0.0.0}
     port: ${oc.env:RAY_SERVE_PORT, 8080}
-    chainlit_port: ${oc.env:CHAINLIT_PORT, 8090}
+    chainlit_port: ${oc.env:CHAINLIT_PORT, 8090}

docs/content/docs/documentation/data_model.md

@@ -39,8 +39,6 @@ erDiagram
         varchar token UK
         boolean is_admin
         datetime created_at
-        int file_quota
-        int file_count
     }
 
     partition_memberships {
@@ -64,10 +62,8 @@ Stores information about API users and administrators.
 | `token`        | String (unique, hashed) | SHA-256 hash of the user's API token |
 | `is_admin`     | Boolean | Marks system administrator users |
 | `created_at`   | DateTime | Timestamp of creation |
-| `file_quota`   | Integer (nullable) | Max files allowed for that user |
-| `file_count`| Integer (default=0) | Number of uploaded files.|
-**Relationships**
-- `memberships`: one-to-many → `PartitionMembership`
+
+**Relationships:** `memberships` one-to-many → `PartitionMembership`
 
 ---
 
@@ -245,17 +241,6 @@ flowchart LR
 
 ---
 
-## **File Quotas**
-Limits the number of files a user can upload (indexed files + pending tasks).
-
-- Admins always have unlimited quota and can update quota for a given user
-- `DEFAULT_FILE_QUOTA < 0` to disable quota checking. 
-- `DEFAULT_FILE_QUOTA >= 0` to set a default quota for all users.
-
-The default value `DEFAULT_FILE_QUOTA` is -1, meaning that file quota checking is bypassed.
-
----
-
 ## Token Handling
 
 - Tokens are generated at user creation time (`or-<random hex>`)

docs/content/docs/documentation/env_vars.md

@@ -155,6 +155,7 @@ The PostgreSQL database is configured using the following environment variables:
 | `POSTGRES_USER` | str | root | Username for database authentication |
 | `POSTGRES_PASSWORD` | str | root_password | Password for database authentication |
 
+
 ## Chat Pipeline
 ### LLM & VLM Configuration
 The system uses two types of language models:
@@ -417,11 +418,11 @@ The following environment variables configure the FastAPI server and control acc
 | `APP_PORT` | `number` | `8000` | Port number on which the FastAPI application listens for incoming requests. |
 | `AUTH_TOKEN` | `string` | `EMPTY` | An authentication token is required to access protected API endpoints. By default, this token corresponds to the API key of the created admin (see [Admin Bootstrapping](/openrag/documentation/user_auth/#2-admin-bootstrapping)). If left empty, authentication is disabled. |
 | `SUPER_ADMIN_MODE` | `boolean` | `false` | Enables super admin privileges when set to `true`, [granting unrestricted access](/openrag/documentation/data_model/#access-control) to all operations and bypassing standard access controls. This is for debugging |
-| `DEFAULT_FILE_QUOTA` | `int` | `-1` | Default per-user file quota. `<0` disables quotas globally; `>=0` sets the default limit when a user has no explicit quota. |
 |`API_NUM_WORKERS`|`int`|1|Number of uvicorn workers|
 | `PREFERRED_URL_SCHEME` | `string` | `null` | URL scheme (`http` or `https`) used when generating URLs in API responses (e.g., `task_status_url`). When running behind a reverse proxy that terminates SSL, set this to `https` to ensure generated URLs use the correct scheme. If unset, the scheme from the incoming request is used. |
 
 
+
 :::caution[Security Notice]
 Always set a strong **`AUTH_TOKEN`** in production environments. Never leave it empty or use default values in production deployments.
 :::
@@ -438,4 +439,4 @@ Always set a strong **`AUTH_TOKEN`** in production environments. Never leave it
 ### Chainlit
 [See this](/openrag/documentation/setup_chainlit_ui_auth/) for chainlit authentification
 
-[See this](/openrag/documentation/chainlit_data_persistency/) for chainlit data persistency
+[See this](/openrag/documentation/chainlit_data_persistency/) for chainlit data persistency

extern/indexer-ui

@@ -146,15 +146,15 @@ async def insert_documents(self, chunks, user):
         await vectordb.async_add_documents.remote(chunks, user)
 
     @ray.method(concurrency_group="delete")
-    async def delete_file(self, file_id: str, partition: str, user: dict) -> bool:
+    async def delete_file(self, file_id: str, partition: str) -> bool:
         log = self.logger.bind(file_id=file_id, partition=partition)
         vectordb = ray.get_actor("Vectordb", namespace="openrag")
         if not self.enable_insertion:
             log.error("Vector database is not enabled, but delete_file was called.")
             return False
 
         try:
-            await vectordb.delete_file.remote(file_id, partition, user_id=user.get("id"))
+            await vectordb.delete_file.remote(file_id, partition)
             log.info("Deleted file from partition.", file_id=file_id, partition=partition)
 
         except Exception as e:
@@ -180,7 +180,7 @@ async def update_file_metadata(
             for doc in docs:
                 doc.metadata.update(metadata)
 
-            await self.delete_file(file_id, partition, user=user)
+            await self.delete_file(file_id, partition)
             await vectordb.async_add_documents.remote(docs, user=user)
 
             log.info("Metadata updated for file.")
@@ -380,16 +380,3 @@ async def get_pool_info(self) -> dict[str, int]:
             "max_tasks_per_worker": MAX_TASKS_PER_WORKER,
             "total_capacity": POOL_SIZE * MAX_TASKS_PER_WORKER,
         }
-
-    @ray.method(concurrency_group="queue_info")
-    async def get_user_pending_task_count(self, user_id: int) -> int:
-        """Count tasks for a user that are not yet COMPLETED or FAILED."""
-        async with self.lock:
-            task_ids = self.user_index.get(user_id, set())
-            pending_states = {"QUEUED", "SERIALIZING", "CHUNKING", "INSERTING"}
-            count = 0
-            for tid in task_ids:
-                info = self.tasks.get(tid)
-                if info and info.state in pending_states:
-                    count += 1
-            return count

openrag/components/indexer/indexer.py

@@ -3,7 +3,6 @@
 import secrets
 from datetime import datetime
 
-from config import load_config
 from sqlalchemy import (
     JSON,
     Boolean,
@@ -16,7 +15,6 @@
     String,
     UniqueConstraint,
     create_engine,
-    func,
     text,
 )
 from sqlalchemy.orm import (
@@ -32,9 +30,6 @@
 from utils.logger import get_logger
 
 logger = get_logger()
-config = load_config()
-
-DEFAULT_FILE_QUOTA = config.rdb.get("default_file_quota", -1)
 
 Base = declarative_base()
 
@@ -116,8 +111,7 @@ class User(Base):
     token = Column(String, unique=True, nullable=True, index=True)
     is_admin = Column(Boolean, default=False, nullable=False)
     created_at = Column(DateTime, default=datetime.now, nullable=False)
-    file_quota = Column(Integer, nullable=True, default=None)
-    file_count = Column(Integer, nullable=False, default=0)
+
     memberships = relationship("PartitionMembership", back_populates="user", cascade="all, delete-orphan")
 
 
@@ -157,7 +151,6 @@ def __init__(self, database_url: str, logger=logger):
             self.Session = sessionmaker(bind=self.engine)
             AUTH_TOKEN = os.getenv("AUTH_TOKEN")
             self._ensure_admin_user(AUTH_TOKEN)
-            self.file_quota_per_user = DEFAULT_FILE_QUOTA
 
         except Exception as e:
             raise VDBConnectionError(
@@ -260,12 +253,6 @@ def add_file_to_partition(
                 )
 
                 session.add(file)
-
-                # Increment file_count for the user
-                user = session.query(User).filter(User.id == user_id).first()
-                if user:
-                    user.file_count = User.file_count + 1
-
                 session.commit()
                 log.info("Added file successfully")
                 return True
@@ -274,7 +261,7 @@ def add_file_to_partition(
                 log.exception("Error adding file to partition")
                 raise
 
-    def remove_file_from_partition(self, file_id: str, partition: str, user_id: int):
+    def remove_file_from_partition(self, file_id: str, partition: str):
         """Remove a file from its partition - Optimized without join"""
         log = self.logger.bind(file_id=file_id, partition=partition)
         with self.Session() as session:
@@ -283,12 +270,6 @@ def remove_file_from_partition(self, file_id: str, partition: str, user_id: int)
                 file = session.query(File).filter(File.file_id == file_id, File.partition_name == partition).first()
                 if file:
                     session.delete(file)
-
-                    # Decrement file_count for the user
-                    user = session.query(User).filter(User.id == user_id).first()
-                    if user and user.file_count > 0:
-                        user.file_count = func.greatest(0, User.file_count - 1)
-
                     session.commit()
                     log.info(f"Removed file {file_id} from partition {partition}")
                     return True
@@ -299,20 +280,11 @@ def remove_file_from_partition(self, file_id: str, partition: str, user_id: int)
                 log.error(f"Error removing file: {e}")
                 raise e
 
-    def delete_partition(self, partition: str, user_id: int):
+    def delete_partition(self, partition: str):
         """Delete a partition and all its files"""
         with self.Session() as session:
             partition_obj = session.query(Partition).filter_by(partition=partition).first()
             if partition_obj:
-                # Count files in the partition before deletion
-                file_count = session.query(File).filter(File.partition_name == partition).count()
-
-                # Decrement file_count for the user
-                if file_count > 0:
-                    user = session.query(User).filter(User.id == user_id).first()
-                    if user:
-                        user.file_count = func.greatest(0, User.file_count - file_count)
-
                 session.delete(partition_obj)  # Will delete all files due to cascade
                 session.commit()
                 self.logger.info("Deleted partition", partition=partition)
@@ -359,22 +331,17 @@ def create_user(
         display_name: str | None = None,
         external_user_id: str | None = None,
         is_admin: bool = False,
-        file_quota: int | None = None,
     ) -> dict:
         """Create a user and generate an API token for them."""
         with self.Session() as s:
             token = f"or-{secrets.token_hex(16)}"
             hashed_token = self.hash_token(token)
 
-            if self.file_quota_per_user > 0 and file_quota is None:
-                file_quota = self.file_quota_per_user  # default to default quota
-
             user = User(
                 display_name=display_name,
                 external_user_id=external_user_id,
                 token=hashed_token,
                 is_admin=is_admin,
-                file_quota=file_quota,
             )
             s.add(user)
             s.commit()
@@ -386,8 +353,6 @@ def create_user(
                 "external_user_id": user.external_user_id,
                 "token": token,
                 "is_admin": user.is_admin,
-                "file_quota": user.file_quota,
-                "file_count": user.file_count,
             }
 
     def list_users(self) -> list[dict]:
@@ -399,8 +364,6 @@ def list_users(self) -> list[dict]:
                     "display_name": u.display_name,
                     "external_user_id": u.external_user_id,
                     "is_admin": u.is_admin,
-                    "file_quota": u.file_quota,
-                    "file_count": u.file_count,
                     "created_at": u.created_at.isoformat(),
                 }
                 for u in users
@@ -427,8 +390,6 @@ def get_user_by_token(self, token: str) -> dict | None:
                 "display_name": user.display_name,
                 "external_user_id": user.external_user_id,
                 "is_admin": user.is_admin,
-                "file_quota": user.file_quota,
-                "file_count": user.file_count,
                 "memberships": memberships,
             }
 
@@ -452,8 +413,6 @@ def get_user_by_id(self, user_id: int) -> dict | None:
                 "display_name": user.display_name,
                 "external_user_id": user.external_user_id,
                 "is_admin": user.is_admin,
-                "file_quota": user.file_quota,
-                "file_count": user.file_count,
                 "memberships": memberships,
             }
 
@@ -481,8 +440,6 @@ def regenerate_user_token(self, user_id: int) -> dict:
                 "external_user_id": user.external_user_id,
                 "token": new_token,
                 "is_admin": user.is_admin,
-                "file_quota": user.file_quota,
-                "file_count": user.file_count,
             }
 
     # Memberships
@@ -574,29 +531,6 @@ def user_is_partition_member(self, user_id: int, partition: str) -> bool:
         with self.Session() as s:
             return s.query(PartitionMembership).filter_by(user_id=user_id, partition_name=partition).first() is not None
 
-    def update_user_quota(self, user_id: int, file_quota: int | None) -> dict:
-        """
-        Update a user's file quota.
-        - None: Use global default (DEFAULT_FILE_QUOTA env var)
-        - <0: Unlimited
-        - >=0: Specific limit
-        """
-        with self.Session() as s:
-            user = s.query(User).filter(User.id == user_id).first()
-            user.file_quota = file_quota
-            s.commit()
-            self.logger.info(f"Updated file_quota for user {user_id} to {file_quota}")
-            s.refresh(user)
-
-            return {
-                "id": user.id,
-                "display_name": user.display_name,
-                "external_user_id": user.external_user_id,
-                "is_admin": user.is_admin,
-                "file_quota": user.file_quota,
-                "file_count": user.file_count,
-            }
-
     def hash_token(self, token: str) -> str:
         """Return a SHA-256 hash of a token string."""
         return hashlib.sha256(token.encode("utf-8")).hexdigest()