feat: Increase MAX_CONTENT_LENGTH to 50,000 chars and make it configurable

README.md

@@ -109,6 +109,7 @@ make mcp        # Start MCP server
 | `PORT` | `8000` | Server port |
 | `LOG_LEVEL` | `INFO` | Logging level (DEBUG, INFO, WARNING, ERROR) |
 | `LOG_DIR` | `/tmp` | Log file directory |
+| `MAX_CONTENT_LENGTH` | `50000` | Maximum characters to return per scraped article |
 
 ### Get a Serper API Key
 

docker/.env.example

@@ -12,6 +12,9 @@ PORT=8000
 LOG_LEVEL=INFO
 LOG_DIR=/tmp
 
+# Content limits
+MAX_CONTENT_LENGTH=50000
+
 # Rate limiting
 RATE_LIMIT_WINDOW=60
 RATE_LIMIT_MAX_REQUESTS=10

docker/README.md

@@ -53,6 +53,7 @@ docker-compose up
 - `PORT`: Port to run the server on (default: 8000)
 - `LOG_LEVEL`: Logging level (default: INFO)
 - `LOG_DIR`: Directory for log files (default: /tmp)
+- `MAX_CONTENT_LENGTH`: Maximum characters to return per scraped article (default: 50000)
 
 ### Simplified Setup
 

docker/constants.py

@@ -5,6 +5,8 @@
 
 """Constants for WebCat application."""
 
+import os
+
 # Application version
 VERSION = "2.3.1"
 
@@ -22,7 +24,7 @@
 ]
 
 # Content limits
-MAX_CONTENT_LENGTH = 8000
+MAX_CONTENT_LENGTH = int(os.environ.get("MAX_CONTENT_LENGTH", "50000"))
 DEFAULT_SEARCH_RESULTS = 5
 
 # Timeout settings

pyproject.toml

@@ -25,17 +25,13 @@ classifiers = [
 ]
 dependencies = [
     "requests>=2.31.0",
-    "beautifulsoup4>=4.12.0",
-    "readability-lxml>=0.8.1",
-    "lxml_html_clean>=0.1.0",
     "python-dotenv>=1.0.0",
     "fastapi>=0.104.0",
     "uvicorn>=0.24.0",
     "pydantic>=2.4.0",
     "sse-starlette>=1.6.0",
     "httpx>=0.24.0",
     "fastmcp>=2.7.0",
-    "html2text>=2020.1.16",
     "duckduckgo-search>=3.9.0",
     "trafilatura>=1.6.0",
 ]
@@ -166,11 +162,10 @@ show_error_codes = true
 
 [[tool.mypy.overrides]]
 module = [
-    "readability.*",
-    "html2text.*",
     "duckduckgo_search.*",
     "fastmcp.*",
     "sse_starlette.*",
+    "trafilatura.*",
 ]
 ignore_missing_imports = true