v1.2.0: fix bugs, update curl_cffi, deprecate python 3.9

Author: el1s7

README.md

@@ -86,7 +86,7 @@ with requests.Session() as s:
 You can get extra information from the curl response info:
 ```python
 import requests
-from curl_adapter import PyCurlAdapter, CurlInfo
+from curl_adapter import PyCurlAdapter, get_curl_info
 
 with requests.Session() as s:
     s.mount("http://", PyCurlAdapter())
@@ -96,7 +96,7 @@ with requests.Session() as s:
 
     body = response.text
 
-    curl_info: CurlInfo = response.curl_info
+    curl_info = get_curl_info(response)
 
     print(
         curl_info

curl_adapter/__init__.py

@@ -9,10 +9,11 @@
 __all__ = [
     "CurlCffiAdapter",
     "PyCurlAdapter",
-    "CurlInfo"
+    "CurlInfo",
+    "get_curl_info"
 ]
 
 
-from .base_adapter import CurlInfo
+from .base_adapter import CurlInfo, get_curl_info
 from .curl_cffi import CurlCffiAdapter
 from .pycurl import PyCurlAdapter

curl_adapter/base_adapter.py

@@ -81,13 +81,15 @@ class CurlInfo(TypedDict):
 	namelookup_time: int
 	has_used_proxy: int
 
+
+def get_curl_info(response: requests.Response) -> CurlInfo:
+	return response.curl_info
+
 class Response(requests.Response):
 	curl_info: CurlInfo
 	wait_for_body: typing.Callable[[], None]
 
 class BaseCurlAdapter(BaseAdapter):
-	
-	_local = threading.local()
 
 	def __init__(self, 
 		curl_class: typing.Union[curl_cffi.Curl, pycurl.Curl],
@@ -96,7 +98,6 @@ def __init__(self,
 		use_thread_local_curl=True,
 		stream_handler: CurlStreamHandlerBase=None
 	):
-		
 		self.curl_class: typing.Union[curl_cffi.Curl, pycurl.Curl] = curl_class
 		self.debug = debug
 		self.use_curl_content_decoding = use_curl_content_decoding
@@ -106,6 +107,14 @@ def __init__(self,
 		self.stream_handler = (stream_handler or CurlStreamHandler)
 
 		if self.use_thread_local_curl:
+			self._local = threading.local()
+			try:
+				from .stream.handler._thread_env import _THREAD_ENV
+				if _THREAD_ENV == "gevent":
+					from gevent.local import local as gevent_local
+					self._local = gevent_local()
+			except Exception:
+				pass
 			self._local.curl = self.curl_class()
 		else:
 			self._curl = self.curl_class()
@@ -121,16 +130,20 @@ def curl(self) -> typing.Union[curl_cffi.Curl, pycurl.Curl]:
 			return self._local.curl
 		return self._curl
 
-	def reset_curl(self):
+	def reset_curl(self, curl: typing.Union[curl_cffi.Curl, pycurl.Curl, None] = None):
 		'''
 			Close current handle and open a new curl handle
 		'''
-		self.curl.close()
+		if curl is None:
+			curl = self.curl
+		curl.close()
 
 		if self.use_thread_local_curl:
 			self._local.curl = self.curl_class()
+			return self._local.curl
 		else:
 			self._curl = self.curl_class()
+			return self._curl
 
 	def enable_debug(self):
 		if self.debug:
@@ -566,14 +579,14 @@ def set_curl_options(self,
 	def send(
 		self, request: requests.PreparedRequest, stream=False, timeout=None, verify=True, cert=None, proxies=None
 	):
-		self.reset_curl()
+		curl = self.reset_curl()
 
-		self.cert_verify(self.curl, request.url, verify, cert)
+		self.cert_verify(curl, request.url, verify, cert)
 
 		url = self.request_url(request, proxies)
 
 		self.set_curl_options(
-			self.curl,
+			curl,
 			request=request,
 			url=url,
 			timeout=timeout,
@@ -583,29 +596,29 @@ def send(
 		try:
 			# Save headers when received
 			header_buffer = BytesIO()
-			self.curl.setopt(CurlOpt.HEADERDATA, header_buffer)
+			curl.setopt(CurlOpt.HEADERDATA, header_buffer)
 
 			# Callbacks for retrieving & saving curl info object
 			curl_info_dict: CurlInfo = {}
 
 			# Perform curl request with threading, and return body in a 'read' like class type (by simply using Curl.WRITEFUNCTION callback)
 			start_curl_stream = (
 				self.stream_handler(
-					curl_instance=self.curl,
+					curl_instance=curl,
 					callback_after_perform=lambda curl: curl_info_dict.update(self.parse_info(curl)),
 					timeout=timeout,
 					debug=self.debug
 				)
 			).start()
 
 			# Headers are already available
-			curl_info_dict.update(self.parse_info(self.curl, headers_only=True))
+			curl_info_dict.update(self.parse_info(curl, headers_only=True))
 
 			if self.debug:
 				print("[DEBUG] Curl Start Elapsed Time: ", time.time() - a)
 
 			# Headers are available after start, parse them
-			parsed_headers = self.parse_headers(self.curl, header_buffer)
+			parsed_headers = self.parse_headers(curl, header_buffer)
 
 			curl_stream_res = CurlStreamResponse(
 				url=url,
@@ -616,7 +629,7 @@ def send(
 				**parsed_headers
 			)
 
-			return self.build_response(self.curl, curl_stream_res, parsed_headers, request, wait_for_body=start_curl_stream._wait_for_body, curl_info_dict=curl_info_dict)
+			return self.build_response(curl, curl_stream_res, parsed_headers, request, wait_for_body=start_curl_stream._wait_for_body, curl_info_dict=curl_info_dict)
 		except OSError as e:
 			raise ConnectionError(e, request=request)
 

curl_adapter/curl_cffi.py

@@ -191,10 +191,11 @@ def set_curl_options(self, curl, request, url, timeout, proxies):
 				)
 
 	def reset_curl(self):
-		if hasattr(self.curl, 'clean_handles_and_buffers'):
+		curl = self.curl
+		if hasattr(curl, 'clean_handles_and_buffers'):
 			# curl_cffi >= 0.14.0: clean_after_perform() was renamed to clean_handles_and_buffers()
-			self.curl.clean_handles_and_buffers()
-		elif hasattr(self.curl, 'clean_after_perform'):
+			curl.clean_handles_and_buffers()
+		elif hasattr(curl, 'clean_after_perform'):
 			# curl_cffi < 0.14.0
-			self.curl.clean_after_perform()
-		return super().reset_curl()
+			curl.clean_after_perform()
+		return super().reset_curl(curl=curl)

curl_adapter/stream/sockets/curl_cffi_socket.py

@@ -32,63 +32,81 @@
 GEVENT_READ = 1
 GEVENT_WRITE = 2
 
+_TIMER_CALLBACK_RETURNS_INT = ffi.typeof(lib.timer_function).result.cname != "void"
+_SOCKET_CALLBACK_RETURNS_INT = ffi.typeof(lib.socket_function).result.cname != "void"
+
+
+def _timer_callback_return():
+	return 0 if _TIMER_CALLBACK_RETURNS_INT else None
+
+
+def _socket_callback_return():
+	return 0 if _SOCKET_CALLBACK_RETURNS_INT else None
+
+
 @ffi.def_extern()
 def timer_function(curlm, timeout_ms: int, clientp: "GeventCurlCffi"):
+	try:
+		gevent_curl: "GeventCurlCffi" = ffi.from_handle(clientp)
+
+		if gevent_curl._timer:
+			gevent_curl._timer.kill(block=False)
+			gevent_curl._timer = None
+
+		if not gevent_curl._curl_multi:
+			# Don't schedule timers if multi handle is gone.
+			return _timer_callback_return()
 
-	gevent_curl: "GeventCurlCffi" = ffi.from_handle(clientp)
-
-	if gevent_curl._timer:
-		gevent_curl._timer.kill(block=False)
-		gevent_curl._timer = None
-
-	if not gevent_curl._curl_multi:
-		# Don't schedule timers if multi handle is gone.
-		return 0
-
-	if timeout_ms == -1:
-		# A timeout_ms value of -1 means you should delete the timer, we already did.
-		return 0
-
-	if timeout_ms == 0:
-		# immediate timeout; invoke directly
-		gevent_curl._timer = gevent.spawn(
-			gevent_curl._process_data, 
-			CURL_SOCKET_TIMEOUT, 
-			CURL_POLL_NONE
-		)
-	elif timeout_ms > 0:
-		# schedule one timer
-		# spawn a greenlet to run after timeout_ms milliseconds
-		gevent_curl._timer = gevent.spawn_later(
+		if timeout_ms == -1:
+			# A timeout_ms value of -1 means you should delete the timer, we already did.
+			return _timer_callback_return()
+
+		if timeout_ms == 0:
+			# immediate timeout; invoke directly
+			gevent_curl._timer = gevent.spawn(
+				gevent_curl._process_data,
+				CURL_SOCKET_TIMEOUT,
+				CURL_POLL_NONE
+			)
+		elif timeout_ms > 0:
+			# spawn a greenlet to run after timeout_ms milliseconds
+			gevent_curl._timer = gevent.spawn_later(
 				timeout_ms / 1000.0,
 				gevent_curl._process_data,
 				CURL_SOCKET_TIMEOUT,
 				CURL_POLL_NONE,
-		)
-	
-	return 0
-			
+			)
+	except Exception:
+		# Never raise from CFFI callbacks; Windows can show "Python-CFFI error" dialogs.
+		return _timer_callback_return()
+
+	return _timer_callback_return()
+
 @ffi.def_extern()
 def socket_function(curlm, sockfd: int, what: int, clientp: "GeventCurlCffi", data: Any):
-	gevent_curl: "GeventCurlCffi" = ffi.from_handle(clientp)
-	want_read  = bool(what & CURL_POLL_IN)
-	want_write = bool(what & CURL_POLL_OUT)
+	try:
+		gevent_curl: "GeventCurlCffi" = ffi.from_handle(clientp)
+		want_read = bool(what & CURL_POLL_IN)
+		want_write = bool(what & CURL_POLL_OUT)
 
-	# compute the new mask for gevent
-	new_mask = 0
-	if want_read:  new_mask |= GEVENT_READ
-	if want_write: new_mask |= GEVENT_WRITE
+		# compute the new mask for gevent
+		new_mask = 0
+		if want_read:
+			new_mask |= GEVENT_READ
+		if want_write:
+			new_mask |= GEVENT_WRITE
 
-	# teardown if libcurl says “remove”
-	if what & CURL_POLL_REMOVE:
-		gevent_curl._update_watcher(sockfd, 0)
-		return 0
+		if what & CURL_POLL_REMOVE:
+			gevent_curl._update_watcher(sockfd, 0)
+			return _socket_callback_return()
 
-	# otherwise install/update the watcher
-	gevent_curl._update_watcher(sockfd, new_mask)
+		gevent_curl._update_watcher(sockfd, new_mask)
+	except Exception:
+		# Never raise from CFFI callbacks; Windows can show "Python-CFFI error" dialogs.
+		return _socket_callback_return()
+
+	return _socket_callback_return()
 
-	return 0
-	
 class GeventCurlCffi:
 	'''
 		Usage:

pyproject.toml

@@ -5,16 +5,16 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "curl_adapter"
-version = "1.1.0"
+version = "1.2.0"
 description = "A curl HTTP adapter switch for requests library — make browser-like requests with custom TLS fingerprints."
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 license = {file = "LICENSE"}
 authors = [{name = "Elis K.", email = "github@elis.cc"}]
 keywords = ["curl", "requests", "adapter", "tls fingerprint", "pycurl", "curl_cffi", "curl impersonate"]
 dependencies = [
     "requests",
-    "curl_cffi >= 0.11.0",
+    "curl_cffi >= 0.14.0",
     "pycurl >= 7.45.5",
     "brotli"
 ]

tests/test_general.py

@@ -202,3 +202,15 @@ def test_ssl_no_verify(self, curl_adapter):
 			assert r.status_code == 200
 
 
+@pytest.mark.parametrize("adapter_class", [CurlCffiAdapter, PyCurlAdapter])
+def test_thread_local_curl_isolated_per_adapter_instance(adapter_class):
+	adapter_one = adapter_class(use_thread_local_curl=True, stream_handler=CurlStreamHandlerBase)
+	adapter_two = adapter_class(use_thread_local_curl=True, stream_handler=CurlStreamHandlerBase)
+
+	try:
+		assert adapter_one.curl is not adapter_two.curl
+	finally:
+		adapter_one.close()
+		adapter_two.close()
+
+