Merge branch 'dev' into improve-swedish-translation

Author: tjbck

backend/open_webui/config.py

@@ -44,7 +44,7 @@ def filter(self, record: logging.LogRecord) -> bool:
 
 # Function to run the alembic migrations
 def run_migrations():
-    print("Running migrations")
+    log.info("Running migrations")
     try:
         from alembic import command
         from alembic.config import Config
@@ -57,7 +57,7 @@ def run_migrations():
 
         command.upgrade(alembic_cfg, "head")
     except Exception as e:
-        print(f"Error: {e}")
+        log.exception(f"Error running migrations: {e}")
 
 
 run_migrations()
@@ -678,6 +678,10 @@ def oidc_oauth_register(client):
 S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", None)
 S3_KEY_PREFIX = os.environ.get("S3_KEY_PREFIX", None)
 S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", None)
+S3_USE_ACCELERATE_ENDPOINT = (
+    os.environ.get("S3_USE_ACCELERATE_ENDPOINT", "False").lower() == "true"
+)
+S3_ADDRESSING_STYLE = os.environ.get("S3_ADDRESSING_STYLE", None)
 
 GCS_BUCKET_NAME = os.environ.get("GCS_BUCKET_NAME", None)
 GOOGLE_APPLICATION_CREDENTIALS_JSON = os.environ.get(
@@ -1094,7 +1098,7 @@ class BannerModel(BaseModel):
     banners = json.loads(os.environ.get("WEBUI_BANNERS", "[]"))
     banners = [BannerModel(**banner) for banner in banners]
 except Exception as e:
-    print(f"Error loading WEBUI_BANNERS: {e}")
+    log.exception(f"Error loading WEBUI_BANNERS: {e}")
     banners = []
 
 WEBUI_BANNERS = PersistentConfig("WEBUI_BANNERS", "ui.banners", banners)
@@ -1566,6 +1570,18 @@ class BannerModel(BaseModel):
     os.environ.get("GOOGLE_DRIVE_API_KEY", ""),
 )
 
+ENABLE_ONEDRIVE_INTEGRATION = PersistentConfig(
+    "ENABLE_ONEDRIVE_INTEGRATION",
+    "onedrive.enable",
+    os.getenv("ENABLE_ONEDRIVE_INTEGRATION", "False").lower() == "true",
+)
+
+ONEDRIVE_CLIENT_ID = PersistentConfig(
+    "ONEDRIVE_CLIENT_ID",
+    "onedrive.client_id",
+    os.environ.get("ONEDRIVE_CLIENT_ID", ""),
+)
+
 # RAG Content Extraction
 CONTENT_EXTRACTION_ENGINE = PersistentConfig(
     "CONTENT_EXTRACTION_ENGINE",
@@ -1579,6 +1595,18 @@ class BannerModel(BaseModel):
     os.getenv("TIKA_SERVER_URL", "http://tika:9998"),  # Default for sidecar deployment
 )
 
+DOCUMENT_INTELLIGENCE_ENDPOINT = PersistentConfig(
+    "DOCUMENT_INTELLIGENCE_ENDPOINT",
+    "rag.document_intelligence_endpoint",
+    os.getenv("DOCUMENT_INTELLIGENCE_ENDPOINT", ""),
+)
+
+DOCUMENT_INTELLIGENCE_KEY = PersistentConfig(
+    "DOCUMENT_INTELLIGENCE_KEY",
+    "rag.document_intelligence_key",
+    os.getenv("DOCUMENT_INTELLIGENCE_KEY", ""),
+)
+
 RAG_TOP_K = PersistentConfig(
     "RAG_TOP_K", "rag.top_k", int(os.environ.get("RAG_TOP_K", "3"))
 )

backend/open_webui/functions.py

@@ -2,6 +2,7 @@
 import sys
 import inspect
 import json
+import asyncio
 
 from pydantic import BaseModel
 from typing import AsyncGenerator, Generator, Iterator
@@ -76,11 +77,13 @@ async def get_function_models(request):
         if hasattr(function_module, "pipes"):
             sub_pipes = []
 
-            # Check if pipes is a function or a list
-
+            # Handle pipes being a list, sync function, or async function
             try:
                 if callable(function_module.pipes):
-                    sub_pipes = function_module.pipes()
+                    if asyncio.iscoroutinefunction(function_module.pipes):
+                        sub_pipes = await function_module.pipes()
+                    else:
+                        sub_pipes = function_module.pipes()
                 else:
                     sub_pipes = function_module.pipes
             except Exception as e:

backend/open_webui/main.py

@@ -95,6 +95,7 @@
     OLLAMA_API_CONFIGS,
     # OpenAI
     ENABLE_OPENAI_API,
+    ONEDRIVE_CLIENT_ID,
     OPENAI_API_BASE_URLS,
     OPENAI_API_KEYS,
     OPENAI_API_CONFIGS,
@@ -180,6 +181,8 @@
     CHUNK_SIZE,
     CONTENT_EXTRACTION_ENGINE,
     TIKA_SERVER_URL,
+    DOCUMENT_INTELLIGENCE_ENDPOINT,
+    DOCUMENT_INTELLIGENCE_KEY,
     RAG_TOP_K,
     RAG_TEXT_SPLITTER,
     TIKTOKEN_ENCODING_NAME,
@@ -215,11 +218,13 @@
     GOOGLE_PSE_ENGINE_ID,
     GOOGLE_DRIVE_CLIENT_ID,
     GOOGLE_DRIVE_API_KEY,
+    ONEDRIVE_CLIENT_ID,
     ENABLE_RAG_HYBRID_SEARCH,
     ENABLE_RAG_LOCAL_WEB_FETCH,
     ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION,
     ENABLE_RAG_WEB_SEARCH,
     ENABLE_GOOGLE_DRIVE_INTEGRATION,
+    ENABLE_ONEDRIVE_INTEGRATION,
     UPLOAD_DIR,
     # WebUI
     WEBUI_AUTH,
@@ -533,6 +538,8 @@ async def lifespan(app: FastAPI):
 
 app.state.config.CONTENT_EXTRACTION_ENGINE = CONTENT_EXTRACTION_ENGINE
 app.state.config.TIKA_SERVER_URL = TIKA_SERVER_URL
+app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT = DOCUMENT_INTELLIGENCE_ENDPOINT
+app.state.config.DOCUMENT_INTELLIGENCE_KEY = DOCUMENT_INTELLIGENCE_KEY
 
 app.state.config.TEXT_SPLITTER = RAG_TEXT_SPLITTER
 app.state.config.TIKTOKEN_ENCODING_NAME = TIKTOKEN_ENCODING_NAME
@@ -564,6 +571,7 @@ async def lifespan(app: FastAPI):
 app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST = RAG_WEB_SEARCH_DOMAIN_FILTER_LIST
 
 app.state.config.ENABLE_GOOGLE_DRIVE_INTEGRATION = ENABLE_GOOGLE_DRIVE_INTEGRATION
+app.state.config.ENABLE_ONEDRIVE_INTEGRATION = ENABLE_ONEDRIVE_INTEGRATION
 app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL
 app.state.config.GOOGLE_PSE_API_KEY = GOOGLE_PSE_API_KEY
 app.state.config.GOOGLE_PSE_ENGINE_ID = GOOGLE_PSE_ENGINE_ID
@@ -911,7 +919,7 @@ def get_filtered_models(models, user):
 
         return filtered_models
 
-    models = await get_all_models(request)
+    models = await get_all_models(request, user=user)
 
     # Filter out filter pipelines
     models = [
@@ -951,7 +959,7 @@ async def chat_completion(
     user=Depends(get_verified_user),
 ):
     if not request.app.state.MODELS:
-        await get_all_models(request)
+        await get_all_models(request, user=user)
 
     model_item = form_data.pop("model_item", {})
     tasks = form_data.pop("background_tasks", None)
@@ -1146,6 +1154,7 @@ async def get_app_config(request: Request):
                     "enable_admin_export": ENABLE_ADMIN_EXPORT,
                     "enable_admin_chat_access": ENABLE_ADMIN_CHAT_ACCESS,
                     "enable_google_drive_integration": app.state.config.ENABLE_GOOGLE_DRIVE_INTEGRATION,
+                    "enable_onedrive_integration": app.state.config.ENABLE_ONEDRIVE_INTEGRATION,
                 }
                 if user is not None
                 else {}
@@ -1177,6 +1186,7 @@ async def get_app_config(request: Request):
                     "client_id": GOOGLE_DRIVE_CLIENT_ID.value,
                     "api_key": GOOGLE_DRIVE_API_KEY.value,
                 },
+                "onedrive": {"client_id": ONEDRIVE_CLIENT_ID.value},
             }
             if user is not None
             else {}

backend/open_webui/models/chats.py

@@ -1,11 +1,12 @@
+import logging
 import json
 import time
 import uuid
 from typing import Optional
 
 from open_webui.internal.db import Base, get_db
 from open_webui.models.tags import TagModel, Tag, Tags
-
+from open_webui.env import SRC_LOG_LEVELS
 
 from pydantic import BaseModel, ConfigDict
 from sqlalchemy import BigInteger, Boolean, Column, String, Text, JSON
@@ -16,6 +17,8 @@
 # Chat DB Schema
 ####################
 
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["MODELS"])
 
 class Chat(Base):
     __tablename__ = "chat"
@@ -670,7 +673,7 @@ def get_chats_by_user_id_and_search_text(
             # Perform pagination at the SQL level
             all_chats = query.offset(skip).limit(limit).all()
 
-            print(len(all_chats))
+            log.info(f"The number of chats: {len(all_chats)}")
 
             # Validate and return chats
             return [ChatModel.model_validate(chat) for chat in all_chats]
@@ -731,7 +734,7 @@ def get_chat_list_by_user_id_and_tag_name(
             query = db.query(Chat).filter_by(user_id=user_id)
             tag_id = tag_name.replace(" ", "_").lower()
 
-            print(db.bind.dialect.name)
+            log.info(f"DB dialect name: {db.bind.dialect.name}")
             if db.bind.dialect.name == "sqlite":
                 # SQLite JSON1 querying for tags within the meta JSON field
                 query = query.filter(
@@ -752,7 +755,7 @@ def get_chat_list_by_user_id_and_tag_name(
                 )
 
             all_chats = query.all()
-            print("all_chats", all_chats)
+            log.debug(f"all_chats: {all_chats}")
             return [ChatModel.model_validate(chat) for chat in all_chats]
 
     def add_chat_tag_by_id_and_user_id_and_tag_name(
@@ -810,7 +813,7 @@ def count_chats_by_tag_name_and_user_id(self, tag_name: str, user_id: str) -> in
             count = query.count()
 
             # Debugging output for inspection
-            print(f"Count of chats for tag '{tag_name}':", count)
+            log.info(f"Count of chats for tag '{tag_name}': {count}")
 
             return count
 

backend/open_webui/models/feedbacks.py

@@ -118,7 +118,7 @@ def insert_new_feedback(
                 else:
                     return None
             except Exception as e:
-                print(e)
+                log.exception(f"Error creating a new feedback: {e}")
                 return None
 
     def get_feedback_by_id(self, id: str) -> Optional[FeedbackModel]:

backend/open_webui/models/files.py

@@ -119,7 +119,7 @@ def insert_new_file(self, user_id: str, form_data: FileForm) -> Optional[FileMod
                 else:
                     return None
             except Exception as e:
-                print(f"Error creating tool: {e}")
+                log.exception(f"Error inserting a new file: {e}")
                 return None
 
     def get_file_by_id(self, id: str) -> Optional[FileModel]:

backend/open_webui/models/folders.py

@@ -82,7 +82,7 @@ def insert_new_folder(
                 else:
                     return None
             except Exception as e:
-                print(e)
+                log.exception(f"Error inserting a new folder: {e}")
                 return None
 
     def get_folder_by_id_and_user_id(

backend/open_webui/models/functions.py

@@ -105,7 +105,7 @@ def insert_new_function(
                 else:
                     return None
         except Exception as e:
-            print(f"Error creating tool: {e}")
+            log.exception(f"Error creating a new function: {e}")
             return None
 
     def get_function_by_id(self, id: str) -> Optional[FunctionModel]:
@@ -170,7 +170,7 @@ def get_function_valves_by_id(self, id: str) -> Optional[dict]:
                 function = db.get(Function, id)
                 return function.valves if function.valves else {}
             except Exception as e:
-                print(f"An error occurred: {e}")
+                log.exception(f"Error getting function valves by id {id}: {e}")
                 return None
 
     def update_function_valves_by_id(
@@ -202,7 +202,7 @@ def get_user_valves_by_id_and_user_id(
 
             return user_settings["functions"]["valves"].get(id, {})
         except Exception as e:
-            print(f"An error occurred: {e}")
+            log.exception(f"Error getting user values by id {id} and user id {user_id}: {e}")
             return None
 
     def update_user_valves_by_id_and_user_id(
@@ -225,7 +225,7 @@ def update_user_valves_by_id_and_user_id(
 
             return user_settings["functions"]["valves"][id]
         except Exception as e:
-            print(f"An error occurred: {e}")
+            log.exception(f"Error updating user valves by id {id} and user_id {user_id}: {e}")
             return None
 
     def update_function_by_id(self, id: str, updated: dict) -> Optional[FunctionModel]:

backend/open_webui/models/models.py

@@ -166,7 +166,7 @@ def insert_new_model(
                 else:
                     return None
         except Exception as e:
-            print(e)
+            log.exception(f"Failed to insert a new model: {e}")
             return None
 
     def get_all_models(self) -> list[ModelModel]:
@@ -246,8 +246,7 @@ def update_model_by_id(self, id: str, model: ModelForm) -> Optional[ModelModel]:
                 db.refresh(model)
                 return ModelModel.model_validate(model)
         except Exception as e:
-            print(e)
-
+            log.exception(f"Failed to update the model by id {id}: {e}")
             return None
 
     def delete_model_by_id(self, id: str) -> bool:

backend/open_webui/models/tags.py

@@ -61,7 +61,7 @@ def insert_new_tag(self, name: str, user_id: str) -> Optional[TagModel]:
                 else:
                     return None
             except Exception as e:
-                print(e)
+                log.exception(f"Error inserting a new tag: {e}")
                 return None
 
     def get_tag_by_name_and_user_id(