chore: Add e2e tests for playwright templates and pip based templates (#1136)

### Description
- Add e2e tests for playwright templates
- Add e2e tests for pip-based templates
- Fix template issue with pip template that has additional dependencies
- Separate all e2e tests in CI to run in it's own runner to allow
selective re-triggers (e2e tests work on real network and it will always
have some flakiness due to that)
- Reduce `max_requests_per_crawl` to 10 in all templates to be aligned
with doc examples and to make the tests faster.

### Issues

- Closes: #1109

Author: Pijukatel

.github/workflows/templates_e2e_tests.yaml

@@ -2,16 +2,19 @@ name: Templates end-to-end tests
 
 on:
   workflow_dispatch:
-    secrets:
-      APIFY_TEST_USER_API_TOKEN:
-        description: API token of the Python testing user on Apify
-        required: true
+  schedule:
+    - cron: '0 6 * * *'
 
 jobs:
   end_to_end_tests:
     name: End-to-end tests
     strategy:
       fail-fast: false
+      max-parallel: 12
+      matrix:
+        crawler-type: ["playwright_camoufox", "playwright", "parsel", "beautifulsoup"]
+        http-client: [ "httpx", "curl_impersonate"]
+        package-manager: ["pip", "uv", "poetry"]
 
     runs-on: "ubuntu-latest"
     env:
@@ -44,10 +47,11 @@ jobs:
         with:
           python-version: ${{ env.python-version }}
 
+      # Sync the project, but no need to install the browsers into the test runner environment.
       - name: Install Python dependencies
-        run: make install-dev
+        run: make install-sync
 
       - name: Run templates end-to-end tests
-        run: make e2e-templates-tests
+        run: make e2e-templates-tests args="-m ${{ matrix.http-client }} and ${{ matrix.crawler-type }} and ${{ matrix.package-manager }}"
         env:
           APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_API_TOKEN }}

Makefile

@@ -1,14 +1,17 @@
-.PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov \
-	integration-tests format check-code build-api-reference run-docs
+.PHONY: clean install-sync install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov \
+	e2e-templates-tests format check-code build-api-reference run-docs
 
 # This is default for local testing, but GitHub workflows override it to a higher value in CI
 E2E_TESTS_CONCURRENCY = 1
 
 clean:
 	rm -rf .mypy_cache .pytest_cache .ruff_cache build dist htmlcov .coverage
 
-install-dev:
+install-sync:
 	uv sync --all-extras
+
+install-dev:
+	make install-sync
 	uv run pre-commit install
 	uv run playwright install
 
@@ -32,8 +35,8 @@ unit-tests:
 unit-tests-cov:
 	uv run pytest --numprocesses=auto --verbose --cov=src/crawlee --cov-report=html tests/unit
 
-e2e-templates-tests:
-	uv run pytest --numprocesses=$(E2E_TESTS_CONCURRENCY) --verbose tests/e2e/project_template
+e2e-templates-tests $(args):
+	uv run pytest --numprocesses=$(E2E_TESTS_CONCURRENCY) --verbose tests/e2e/project_template "$(args)"
 
 format:
 	uv run ruff check --fix

src/crawlee/project_template/templates/main_beautifulsoup.py

@@ -7,6 +7,6 @@
 # % block instantiation
 crawler = BeautifulSoupCrawler(
     request_handler=router,
-    max_requests_per_crawl=50,
+    max_requests_per_crawl=10,
     {{ self.http_client_instantiation() }})
 # % endblock

src/crawlee/project_template/templates/main_parsel.py

@@ -7,6 +7,6 @@
 # % block instantiation
 crawler = ParselCrawler(
     request_handler=router,
-    max_requests_per_crawl=50,
+    max_requests_per_crawl=10,
     {{ self.http_client_instantiation() }})
 # % endblock

src/crawlee/project_template/templates/main_playwright.py

@@ -8,6 +8,6 @@
 crawler = PlaywrightCrawler(
     request_handler=router,
     headless=True,
-    max_requests_per_crawl=50,
+    max_requests_per_crawl=10,
     {{ self.http_client_instantiation() }})
 # % endblock

src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile

@@ -72,8 +72,8 @@ RUN echo "Python version:" \
  && cat requirements.txt | \
  # Replace playwright version so that it matches whatever is pre-installed in the image (the `hash` checks if playwright is installed)
     sed "s/^playwright==\(.*\)/playwright==$(hash playwright 2>/dev/null && (playwright --version | cut -d ' ' -f 2) || echo '\1')/" | \
- # Install everything using pip (ignore dependency checks - the lockfile is correct, period)
-    pip install -r /dev/stdin --no-dependencies \
+ # Install everything using pip
+    pip install -r /dev/stdin \
  && echo "All installed Python packages:" \
  && pip freeze
 # % elif cookiecutter.package_manager == 'manual'

src/crawlee/project_template/{{cookiecutter.project_name}}/pyproject.toml

@@ -19,7 +19,7 @@ requires-python = ">=3.9,<4.0"
 dependencies = [
     "crawlee[{{ extras|join(',') }}]",
     # % if cookiecutter.crawler_type == 'playwright-camoufox'
-    "camoufox[geoip]>=0.4.5",
+    "camoufox[geoip]~=0.4.5",
     # % endif
     # % if cookiecutter.enable_apify_integration
     "apify",

src/crawlee/project_template/{{cookiecutter.project_name}}/requirements.txt

@@ -1,5 +1,5 @@
 # % if cookiecutter.crawler_type == 'playwright-camoufox'
-camoufox
+camoufox[geoip]~=0.4.5
 # % set extras = ['playwright']
 # % else
 # % set extras = [cookiecutter.crawler_type]

tests/e2e/conftest.py

@@ -2,11 +2,27 @@
 from pathlib import Path
 
 import pytest
+from _pytest.config import Config
 from filelock import FileLock
 
 _CRAWLEE_ROOT_PATH = Path(__file__).parent.parent.parent.resolve()
 
 
+def pytest_configure(config: Config) -> None:
+    for marker in [
+        'httpx',
+        'curl_impersonate',
+        'playwright',
+        'playwright_camoufox',
+        'parsel',
+        'beautifulsoup',
+        'uv',
+        'poetry',
+        'pip',
+    ]:
+        config.addinivalue_line('markers', f'{marker}: Integration test parameter marker.')
+
+
 @pytest.fixture(scope='session')
 def crawlee_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) -> Path:
     """Build the package wheel if it hasn't been built yet, and return the path to the wheel."""

tests/e2e/project_template/test_static_crawlers_templates.py

@@ -2,24 +2,50 @@
 import re
 import subprocess
 from pathlib import Path
+from typing import Literal
 
 import pytest
 from apify_client import ApifyClientAsync
 from cookiecutter.main import cookiecutter
 
 from crawlee._cli import default_start_url, template_directory
 from crawlee._utils.crypto import crypto_random_object_id
-from tests.e2e.project_template.utils import patch_crawlee_version_in_pyproject_toml_based_project
+from tests.e2e.project_template.utils import patch_crawlee_version_in_project
 
 # To run these tests locally, make sure you have apify-cli installed and available in the path.
 # https://docs.apify.com/cli/docs/installation
 
 
-@pytest.mark.parametrize('http_client', ['httpx', 'curl-impersonate'])
-@pytest.mark.parametrize('crawler_type', ['parsel', 'beautifulsoup'])
-@pytest.mark.parametrize('package_manager', ['uv', 'poetry'])
+@pytest.mark.parametrize(
+    'crawler_type',
+    [
+        pytest.param('playwright-camoufox', marks=pytest.mark.playwright_camoufox),
+        pytest.param('playwright', marks=pytest.mark.playwright),
+        pytest.param('parsel', marks=pytest.mark.parsel),
+        pytest.param('beautifulsoup', marks=pytest.mark.beautifulsoup),
+    ],
+)
+@pytest.mark.parametrize(
+    'http_client',
+    [
+        pytest.param('httpx', marks=pytest.mark.httpx),
+        pytest.param('curl-impersonate', marks=pytest.mark.curl_impersonate),
+    ],
+)
+@pytest.mark.parametrize(
+    'package_manager',
+    [
+        pytest.param('pip', marks=pytest.mark.pip),
+        pytest.param('uv', marks=pytest.mark.uv),
+        pytest.param('poetry', marks=pytest.mark.poetry),
+    ],
+)
 async def test_static_crawler_actor_at_apify(
-    tmp_path: Path, crawlee_wheel_path: Path, package_manager: str, crawler_type: str, http_client: str
+    tmp_path: Path,
+    crawlee_wheel_path: Path,
+    package_manager: Literal['pip', 'uv', 'poetry'],
+    crawler_type: str,
+    http_client: str,
 ) -> None:
     # Generate new actor name
     actor_name = f'crawlee-python-template-e2e-test-{crypto_random_object_id(8).lower()}'
@@ -40,8 +66,8 @@ async def test_static_crawler_actor_at_apify(
         output_dir=tmp_path,
     )
 
-    patch_crawlee_version_in_pyproject_toml_based_project(
-        project_path=tmp_path / actor_name, wheel_path=crawlee_wheel_path
+    patch_crawlee_version_in_project(
+        project_path=tmp_path / actor_name, wheel_path=crawlee_wheel_path, package_manager=package_manager
     )
 
     # Build actor using sequence of cli commands as the user would
@@ -56,16 +82,19 @@ async def test_static_crawler_actor_at_apify(
     build_process = subprocess.run(['apify', 'push'], capture_output=True, check=False, cwd=tmp_path / actor_name)  # noqa: ASYNC221, S603, S607
     # Get actor ID from build log
     actor_id_regexp = re.compile(r'https:\/\/console\.apify\.com\/actors\/(.*)#\/builds\/\d*\.\d*\.\d*')
-    # Why is it in stderr and not in stdout???
-    actor_id = re.findall(actor_id_regexp, build_process.stderr.decode())[0]
+
+    if match := re.findall(actor_id_regexp, build_process.stderr.decode()):
+        actor_id = match[0]
+    else:
+        raise AssertionError(f'Failed to find actor id in build log: {build_process.stderr.decode()}')
 
     client = ApifyClientAsync(token=os.getenv('APIFY_TEST_USER_API_TOKEN'))
     actor = client.actor(actor_id)
 
     # Run actor
     try:
         assert build_process.returncode == 0
-        started_run_data = await actor.start()
+        started_run_data = await actor.start(memory_mbytes=8192)
         actor_run = client.run(started_run_data['id'])
 
         finished_run_data = await actor_run.wait_for_finish()
@@ -80,6 +109,6 @@ async def test_static_crawler_actor_at_apify(
     assert finished_run_data
     assert finished_run_data['status'] == 'SUCCEEDED', additional_run_info
     assert (
-        'Crawler.stop() was called with following reason: The crawler has reached its limit of 50 requests per crawl.'
+        'Crawler.stop() was called with following reason: The crawler has reached its limit of 10 requests per crawl.'
     ) in actor_run_log, additional_run_info
-    assert int(re.findall(r'requests_finished\s*│\s*(\d*)', actor_run_log)[-1]) >= 50, additional_run_info
+    assert int(re.findall(r'requests_finished\s*│\s*(\d*)', actor_run_log)[-1]) >= 10, additional_run_info