feat: optimize resource filtering, enhance ad/tracker blocking and cleanup docs

- Refactor BrowserManager to dynamically block resources based on avoid_css and text_mode
- Align text_mode behavior with community standards (no forced CSS blocking)
- Add Top 20 curated ad and tracker patterns for performance
- Restore and translate permanent browser logs in crawler_pool.py
- Clean up models.py schema annotations and server.py docstrings
- Add unit and functional tests for filtering flags

Author: dylan.min

crawl4ai/browser_manager.py

@@ -878,79 +878,29 @@ async def create_browser_context(self, crawlerRunConfig: CrawlerRunConfig = None
         }
         proxy_settings = {"server": self.config.proxy} if self.config.proxy else None
 
-        blocked_extensions = [
+        # Define resource categories
+        css_extensions = ["css", "less", "scss", "sass"]
+        static_extensions = [
             # Images
-            "jpg",
-            "jpeg",
-            "png",
-            "gif",
-            "webp",
-            "svg",
-            "ico",
-            "bmp",
-            "tiff",
-            "psd",
+            "jpg", "jpeg", "png", "gif", "webp", "svg", "ico", "bmp", "tiff",
             # Fonts
-            "woff",
-            "woff2",
-            "ttf",
-            "otf",
-            "eot",
-            # Styles
-            "css", "less", "scss", "sass",
+            "woff", "woff2", "ttf", "otf", "eot",
             # Media
-            "mp4",
-            "webm",
-            "ogg",
-            "avi",
-            "mov",
-            "wmv",
-            "flv",
-            "m4v",
-            "mp3",
-            "wav",
-            "aac",
-            "m4a",
-            "opus",
-            "flac",
-            # Documents
-            "pdf",
-            "doc",
-            "docx",
-            "xls",
-            "xlsx",
-            "ppt",
-            "pptx",
-            # Archives
-            "zip",
-            "rar",
-            "7z",
-            "tar",
-            "gz",
-            # Scripts and data
-            "xml",
-            "swf",
-            "wasm",
+            "mp4", "webm", "ogg", "mp3", "wav", "aac", "flac",
+            # Documents & Archives
+            "pdf", "doc", "docx", "xls", "xlsx", "zip", "rar", "7z", "tar", "gz",
+            # Other
+            "xml", "swf", "wasm"
         ]
 
-        # Ad and Tracker patterns
+        # Ad and Tracker patterns (Top 20 curated from uBlock sources for performance)
         ad_tracker_patterns = [
-            "**/google-analytics.com/**",
-            "**/googletagmanager.com/**",
-            "**/googlesyndication.com/**",
-            "**/doubleclick.net/**",
-            "**/adservice.google.com/**",
-            "**/adsystem.com/**",
-            "**/adzerk.net/**",
-            "**/adnxs.com/**",
-            "**/ads.linkedin.com/**",
-            "**/facebook.net/**",
-            "**/analytics.twitter.com/**",
-            "**/t.co/**",
-            "**/hotjar.com/**",
-            "**/clarity.ms/**",
-            "**/scorecardresearch.com/**",
-            "**/pixel.wp.com/**",
+            "**/google-analytics.com/**", "**/googletagmanager.com/**", "**/googlesyndication.com/**",
+            "**/doubleclick.net/**", "**/adservice.google.com/**", "**/adsystem.com/**",
+            "**/adzerk.net/**", "**/adnxs.com/**", "**/ads.linkedin.com/**", "**/facebook.net/**",
+            "**/analytics.twitter.com/**", "**/t.co/**", "**/ads-twitter.com/**",
+            "**/hotjar.com/**", "**/clarity.ms/**", "**/scorecardresearch.com/**", "**/pixel.wp.com/**",
+            "**/amazon-adsystem.com/**", "**/mixpanel.com/**", "**/segment.com/**"
         ]
 
         # Common context settings
@@ -1006,10 +956,15 @@ async def create_browser_context(self, crawlerRunConfig: CrawlerRunConfig = None
         # Create and return the context with all settings
         context = await self.browser.new_context(**context_settings)
 
-        # Apply resource filtering based on config
-        if self.config.avoid_css or self.config.text_mode:
-            # Create and apply route patterns for each extension
-            for ext in blocked_extensions:
+        # Apply resource filtering based on config (Dynamic addition)
+        to_block = []
+        if self.config.avoid_css:
+            to_block += css_extensions
+        if self.config.text_mode:
+            to_block += static_extensions
+            
+        if to_block:
+            for ext in to_block:
                 await context.route(f"**/*.{ext}", lambda route: route.abort())
 
         if self.config.avoid_ads:

crawl4ai/models.py

@@ -151,7 +151,7 @@ class CrawlResult(BaseModel):
     redirected_url: Optional[str] = None
     network_requests: Optional[List[Dict[str, Any]]] = None
     console_messages: Optional[List[Dict[str, Any]]] = None
-    tables: List[Dict] = Field(default_factory=list)  # NEW – [{headers,rows,caption,summary}]
+    tables: List[Dict] = Field(default_factory=list)
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 

deploy/docker/crawler_pool.py

@@ -50,7 +50,7 @@ async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
         if not PERMANENT_BROWSER_DISABLED and PERMANENT and _is_default_config(sig):
             LAST_USED[sig] = time.time()
             USAGE_COUNT[sig] = USAGE_COUNT.get(sig, 0) + 1
-            # logger.info("🔥 Using permanent browser")
+            logger.info("🔥 Using permanent browser")
             return PERMANENT
 
         # Check hot pool
@@ -172,7 +172,7 @@ async def janitor():
         mem_pct = get_container_memory_percent()
 
         # Adaptive intervals and TTLs
-        # 严格遵循 BASE_IDLE_TTL，不再做 hot_ttl = ttl * 2 的放大
+        # Strictly follow BASE_IDLE_TTL without multipliers
         if mem_pct > 80:
             interval, cold_ttl, hot_ttl = 10, 30, 60
         elif mem_pct > 60:

tests/browser/test_resource_filtering.py

@@ -0,0 +1,55 @@
+import asyncio
+import os
+import sys
+import pytest
+
+# Add the project root to Python path if running directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+# Create a logger for clear terminal output
+logger = AsyncLogger(verbose=True, log_file=None)
+
+@pytest.mark.asyncio
+async def test_resource_filtering_launch():
+    """Functional test to ensure browser launches correctly with filtering flags enabled."""
+    browser_config = BrowserConfig(
+        headless=True,
+        avoid_ads=True,
+        avoid_css=True,
+        text_mode=True
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Simple crawl to verify functionality
+        result = await crawler.arun(
+            url="https://example.com",
+            config=CrawlerRunConfig(cache_mode="bypass")
+        )
+        assert result.success
+        logger.success("Browser launched and crawled successfully with filtering flags")
+
+@pytest.mark.asyncio
+async def test_avoid_css_only():
+    """Test avoid_css without text_mode."""
+    browser_config = BrowserConfig(
+        headless=True,
+        avoid_css=True,
+        text_mode=False
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://example.com",
+            config=CrawlerRunConfig(cache_mode="bypass")
+        )
+        assert result.success
+        logger.success("Browser launched and crawled successfully with avoid_css only")
+
+if __name__ == "__main__":
+    asyncio.run(test_resource_filtering_launch())
+    asyncio.run(test_avoid_css_only())
+

tests/general/test_cache_context.py

@@ -1,7 +1,9 @@
 import asyncio
+import pytest
 from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
 from playwright.async_api import Page, BrowserContext
 
+@pytest.mark.asyncio
 async def test_reuse_context_by_config():
     # We will store each context ID in these maps to confirm reuse
     context_ids_for_A = []

tests/unit/test_config_flags.py

@@ -0,0 +1,33 @@
+import pytest
+from crawl4ai.async_configs import BrowserConfig
+
+def test_browser_config_filtering_flags():
+    """Test that BrowserConfig correctly stores the new filtering flags."""
+    # Default values
+    config = BrowserConfig()
+    assert config.avoid_ads is False
+    assert config.avoid_css is False
+    
+    # Custom values
+    config = BrowserConfig(avoid_ads=True, avoid_css=True)
+    assert config.avoid_ads is True
+    assert config.avoid_css is True
+    
+    # Check to_dict / from_kwargs parity
+    config_dict = config.to_dict()
+    assert config_dict["avoid_ads"] is True
+    assert config_dict["avoid_css"] is True
+    
+    new_config = BrowserConfig.from_kwargs(config_dict)
+    assert new_config.avoid_ads is True
+    assert new_config.avoid_css is True
+
+def test_browser_config_clone():
+    """Test that cloning BrowserConfig preserves the new flags."""
+    config = BrowserConfig(avoid_ads=True, avoid_css=False)
+    cloned = config.clone(avoid_css=True)
+    
+    assert cloned.avoid_ads is True
+    assert cloned.avoid_css is True
+    assert config.avoid_css is False # Original remains unchanged
+