tests are fixed

Author: vdusek

.pre-commit-config.yaml

@@ -19,8 +19,8 @@ repos:
         language: system
         pass_filenames: false
 
-      - id: check-version-conflict
-        name: Check version conflict
-        entry: make check-version-conflict
+      - id: check-version-availability
+        name: Check version availability
+        entry: make check-version-availability
         language: system
         pass_filenames: false

pyproject.toml

@@ -162,6 +162,7 @@ max-branches = 18
 
 [tool.pytest.ini_options]
 addopts = "-ra"
+asyncio_default_fixture_loop_scope = "function"
 asyncio_mode = "auto"
 timeout = 1200
 

src/apify/_configuration.py

@@ -263,13 +263,13 @@ class Configuration(CrawleeConfiguration):
     ] = 4321
 
     standby_url: Annotated[
-        str | None,
+        str,
         BeforeValidator(validate_http_url),
         Field(
             alias='actor_standby_url',
             description='URL for accessing web servers of Actor runs in Standby mode',
         ),
-    ] = None
+    ] = 'http://localhost'
 
     token: Annotated[
         str | None,

src/apify/scrapy/requests.py

@@ -165,13 +165,7 @@ def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
 
     # Add optional 'headers' field
     if apify_request.headers:
-        if isinstance(cast(Any, apify_request.headers), dict):
-            scrapy_request.headers = Headers(apify_request.headers)
-        else:
-            Actor.log.warning(
-                'apify_request[headers] is not an instance of the dict class, '
-                f'apify_request[headers] = {apify_request.headers}',
-            )
+        scrapy_request.headers |= Headers(apify_request.headers)
 
     # Add optional 'userData' field
     if apify_request.user_data:

tests/unit/scrapy/requests/test_to_apify_request.py

@@ -4,6 +4,8 @@
 from scrapy import Request, Spider
 from scrapy.http.headers import Headers
 
+from crawlee._types import HttpHeaders
+
 from apify.scrapy.requests import to_apify_request
 
 
@@ -36,7 +38,7 @@ def test__to_apify_request__headers(spider: Spider) -> None:
     apify_request = to_apify_request(scrapy_request, spider)
 
     assert apify_request is not None
-    assert apify_request.headers == dict(scrapy_request_headers.to_unicode_dict())
+    assert apify_request.headers == HttpHeaders(scrapy_request_headers.to_unicode_dict())
 
 
 def test__to_apify_request__without_id_and_unique_key(spider: Spider) -> None:

tests/unit/scrapy/requests/test_to_scrapy_request.py

@@ -4,9 +4,9 @@
 
 import pytest
 from scrapy import Request, Spider
-from scrapy.http.headers import Headers
 
 from crawlee import Request as CrawleeRequest
+from crawlee._types import HttpHeaders
 
 from apify.scrapy.requests import to_scrapy_request
 
@@ -47,7 +47,7 @@ def test__to_scrapy_request__without_reconstruction_with_optional_fields(spider:
         method='GET',
         unique_key='https://crawlee.dev',
         id='fvwscO2UJLdr10B',
-        headers={'Authorization': 'Bearer access_token'},
+        headers=HttpHeaders({'Authorization': 'Bearer access_token'}),
         user_data={'some_user_data': 'test'},
     )
 
@@ -58,7 +58,7 @@ def test__to_scrapy_request__without_reconstruction_with_optional_fields(spider:
     assert apify_request.method == scrapy_request.method
     assert apify_request.id == scrapy_request.meta.get('apify_request_id')
     assert apify_request.unique_key == scrapy_request.meta.get('apify_request_unique_key')
-    assert Headers(apify_request.headers) == scrapy_request.headers
+    assert apify_request.headers.get('authorization') == scrapy_request.headers.get('authorization').decode()
     assert apify_request.user_data == scrapy_request.meta.get('userData')
 
 
@@ -91,7 +91,7 @@ def test__to_scrapy_request__with_reconstruction_with_optional_fields(spider: Sp
         method='GET',
         id='fvwscO2UJLdr10B',
         unique_key='https://apify.com',
-        headers={'Authorization': 'Bearer access_token'},
+        headers=HttpHeaders({'Authorization': 'Bearer access_token'}),
         user_data={
             'some_user_data': 'hello',
             'scrapy_request': 'gASVJgIAAAAAAAB9lCiMA3VybJSMEWh0dHBzOi8vYXBpZnkuY29tlIwIY2FsbGJhY2uUTowHZXJy\nYmFja5ROjAdoZWFkZXJzlH2UKEMGQWNjZXB0lF2UQz90ZXh0L2h0bWwsYXBwbGljYXRpb24veGh0\nbWwreG1sLGFwcGxpY2F0aW9uL3htbDtxPTAuOSwqLyo7cT0wLjiUYUMPQWNjZXB0LUxhbmd1YWdl\nlF2UQwJlbpRhQwpVc2VyLUFnZW50lF2UQyNTY3JhcHkvMi4xMS4wICgraHR0cHM6Ly9zY3JhcHku\nb3JnKZRhQw9BY2NlcHQtRW5jb2RpbmeUXZRDDWd6aXAsIGRlZmxhdGWUYXWMBm1ldGhvZJSMA0dF\nVJSMBGJvZHmUQwCUjAdjb29raWVzlH2UjARtZXRhlH2UKIwQYXBpZnlfcmVxdWVzdF9pZJSMD2Z2\nd3NjTzJVSkxkcjEwQpSMGGFwaWZ5X3JlcXVlc3RfdW5pcXVlX2tleZSMEWh0dHBzOi8vYXBpZnku\nY29tlIwQZG93bmxvYWRfdGltZW91dJRHQGaAAAAAAACMDWRvd25sb2FkX3Nsb3SUjAlhcGlmeS5j\nb22UjBBkb3dubG9hZF9sYXRlbmN5lEc/tYIIAAAAAHWMCGVuY29kaW5nlIwFdXRmLTiUjAhwcmlv\ncml0eZRLAIwLZG9udF9maWx0ZXKUiYwFZmxhZ3OUXZSMCWNiX2t3YXJnc5R9lHUu\n',  # noqa: E501
@@ -105,7 +105,7 @@ def test__to_scrapy_request__with_reconstruction_with_optional_fields(spider: Sp
     assert apify_request.method == scrapy_request.method
     assert apify_request.id == scrapy_request.meta.get('apify_request_id')
     assert apify_request.unique_key == scrapy_request.meta.get('apify_request_unique_key')
-    assert Headers(apify_request.headers) == scrapy_request.headers
+    assert apify_request.headers.get('authorization') == scrapy_request.headers.get('authorization').decode()
     assert apify_request.user_data == scrapy_request.meta.get('userData')