do not run sys.exit for scrapy

Author: vdusek

src/apify/_actor.py

@@ -4,6 +4,7 @@
 import os
 import sys
 from datetime import timedelta
+from importlib.util import find_spec
 from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, cast, overload
 
 from lazy_object_proxy import Proxy
@@ -266,10 +267,15 @@ async def finalize() -> None:
         await asyncio.wait_for(finalize(), cleanup_timeout.total_seconds())
         self._is_initialized = False
 
+        # Find out if Scrapy is installed.
+        scrapy_installed = find_spec('scrapy') is not None
+
         if is_running_in_ipython():
             self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in IPython')
         elif os.getenv('PYTEST_CURRENT_TEST', default=False):  # noqa: PLW1508
             self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in an unit test')
+        elif scrapy_installed:
+            self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in Scrapy')
         else:
             sys.exit(exit_code)
 

tests/integration/actor_source_base/requirements.txt

@@ -1,3 +1,2 @@
 # The test fixture will put the Apify SDK wheel path on the next line
 APIFY_SDK_WHEEL_PLACEHOLDER
-scrapy~=2.12.0

tests/integration/conftest.py

@@ -198,6 +198,7 @@ def __call__(
         main_func: Callable | None = None,
         main_py: str | None = None,
         source_files: Mapping[str, str | bytes] | None = None,
+        additional_requirements: list[str] | None = None,
     ) -> Awaitable[ActorClientAsync]:
         """Create a temporary Actor from the given main function or source files.
 
@@ -211,6 +212,7 @@ def __call__(
             main_func: The main function of the Actor.
             main_py: The `src/main.py` file of the Actor.
             source_files: A dictionary of the source files of the Actor.
+            additional_requirements: A list of additional requirements to be added to the `requirements.txt`.
 
         Returns:
             A resource client for the created Actor.
@@ -235,6 +237,7 @@ async def _make_actor(
         main_func: Callable | None = None,
         main_py: str | None = None,
         source_files: Mapping[str, str | bytes] | None = None,
+        additional_requirements: list[str] | None = None,
     ) -> ActorClientAsync:
         if not (main_func or main_py or source_files):
             raise TypeError('One of `main_func`, `main_py` or `source_files` arguments must be specified')
@@ -270,6 +273,16 @@ async def _make_actor(
         actor_source_files = actor_base_source_files.copy()
         actor_source_files.update(source_files)
 
+        if additional_requirements:
+            # Get the current requirements.txt content (as a string).
+            req_content = actor_source_files.get('requirements.txt', '')
+            if isinstance(req_content, bytes):
+                req_content = req_content.decode('utf-8')
+            # Append the additional requirements, each on a new line.
+            additional_reqs = '\n'.join(additional_requirements)
+            req_content = req_content.strip() + '\n' + additional_reqs + '\n'
+            actor_source_files['requirements.txt'] = req_content
+
         # Reformat the source files in a format that the Apify API understands.
         source_files_for_api = []
         for file_name, file_contents in actor_source_files.items():

tests/integration/test_actor_scrapy.py

@@ -22,7 +22,11 @@ async def test_actor_scrapy_title_spider(
         'src/spiders/title.py': read_file('docs/02_guides/code/scrapy_project/src/spiders/title.py'),
     }
 
-    actor = await make_actor('actor-scrapy-title-spider', source_files=actor_source_files)
+    actor = await make_actor(
+        'actor-scrapy-title-spider',
+        source_files=actor_source_files,
+        additional_requirements=['scrapy~=2.12.0'],
+    )
     run_result = await run_actor(
         actor,
         run_input={