Added option to use curl_cffi during search

Author: jecarr

.github/workflows/integration.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
     steps:
     - name: Checkout
       uses: actions/checkout@v2

README.rst

@@ -42,7 +42,7 @@ Community-Driven Development: Tracery is developed with input from the community
 Requirements
 ############
 
-Tracery supports Python 3.9+
+Tracery supports Python 3.10+
 
 Installation
 ############

requirements.txt

@@ -13,3 +13,4 @@ pyyaml~=6.0
 requests[socks]~=2.32
 setproctitle~=1.3
 Werkzeug~=3.1
+curl_cffi~=0.14

searx/poolrequests.py

@@ -4,6 +4,7 @@
 from threading import local
 
 import requests
+from curl_cffi import requests as requests_cffi
 
 from searx import settings
 from searx import logger
@@ -153,6 +154,23 @@ def get_global_proxies():
 
 
 def request(method, url, **kwargs):
+    use_curl_cffi = settings.get('search', dict()).get('use_curl_cffi')
+    if use_curl_cffi:
+        return _request_cffi(method, url)
+
+    return _request(method, url, **kwargs)
+
+
+def _request_cffi(method, url):
+    """Executes the request using curl_cffi."""
+    session = requests_cffi.Session()
+    response = session.request(method=method, url=url, impersonate='chrome')
+    session.close()
+
+    return response
+
+
+def _request(method, url, **kwargs):
     """same as requests/requests/api.py request(...)"""
     time_before_request = time()
 

searx/settings.yml

@@ -20,6 +20,7 @@ search:
     ban_time_on_fail : 5 # ban time in seconds after engine errors
     max_ban_time_on_fail : 120 # max ban time in seconds after engine errors
     prefer_configured_language: False # increase weight of results in configured language in ranking
+    use_curl_cffi : False # whether to use curl_cffi for requests
 
 server:
     port : 9000