Merge branch 'master' into renovate/setuptools-74.x

vdusek · web-flow · commit 61047448933b · 2024-09-09T09:08:14.000+02:00
diff --git a/.github/workflows/check_version_availability.yaml b/.github/workflows/check_version_availability.yaml
@@ -23,7 +23,9 @@ jobs:
           python-version: "3.9"
 
       - name: Install dependencies
-        run: make install-dev
+        run: |
+          pipx install poetry
+          make install-dev
 
       - name: Check version availability
         run: make check-version-availability
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -50,8 +50,10 @@ jobs:
         with:
           python-version: 3.9
 
-      - name: Install Python dependencies
-        run: make install-dev
+      - name: Install dependencies
+        run: |
+          pipx install poetry
+          make install-dev
 
       - name: Build generated API reference
         run: make build-api-reference
diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml
@@ -35,7 +35,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
-        run: make install-dev
+        run: |
+          pipx install poetry
+          make install-dev
 
       - name: Run integration tests
         run: make INTEGRATION_TESTS_CONCURRENCY=8 integration-tests
diff --git a/.github/workflows/lint_and_type_checks.yaml b/.github/workflows/lint_and_type_checks.yaml
@@ -25,7 +25,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
-        run: make install-dev
+        run: |
+          pipx install poetry
+          make install-dev
 
       - name: Run lint
         run: make lint
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -69,7 +69,9 @@ jobs:
           python-version: 3.9
 
       - name: Install dependencies
-        run: make install-dev
+        run: |
+          pipx install poetry
+          make install-dev
 
       - # Determine if this is a prerelease or latest release
         name: Determine release type
diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml
@@ -26,7 +26,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
-        run: make install-dev
+        run: |
+          pipx install poetry
+          make install-dev
 
       - name: Run unit tests
         run: make unit-tests
diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,9 @@ __pycache__
 .envrc
 .python-version
 
+poetry.toml
+poetry.lock
+
 *.egg-info/
 *.egg
 dist/
diff --git a/Makefile b/Makefile
@@ -9,7 +9,6 @@ clean:
 	rm -rf .mypy_cache .pytest_cache .ruff_cache build dist htmlcov .coverage
 
 install-dev:
-	python3 -m pip install --upgrade pip poetry
 	poetry install --all-extras
 	poetry run pre-commit install
 
@@ -35,7 +34,7 @@ unit-tests-cov:
 	poetry run pytest --numprocesses=auto --verbose --cov=src/apify --cov-report=html tests/unit
 
 integration-tests:
-	poetry run pytest --numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) tests/integration
+	poetry run pytest --numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) --verbose tests/integration
 
 format:
 	poetry run ruff check --fix $(DIRS_WITH_CODE)
@@ -52,7 +51,7 @@ check-changelog-entry:
 	poetry run python scripts/check_version_in_changelog.py
 
 build-api-reference:
-	cd website && ./build_api_reference.sh
+	cd website && poetry run ./build_api_reference.sh
 
 run-doc: build-api-reference
 	cd website && npm clean-install && npm run start
diff --git a/pyproject.toml b/pyproject.toml
@@ -58,7 +58,7 @@ websockets = ">=10.0"
 
 [tool.poetry.group.dev.dependencies]
 build = "~1.2.0"
-filelock = "~3.15.0"
+filelock = "~3.16.0"
 griffe = "~1.2.0"
 mypy = "~1.11.0"
 pre-commit = "~3.8.0"
diff --git a/src/apify/__init__.py b/src/apify/__init__.py
@@ -1,6 +1,7 @@
 from importlib import metadata
 
 from apify_shared.consts import WebhookEventType
+from crawlee import Request
 from crawlee.events._types import Event
 
 from apify._actor import Actor
@@ -16,6 +17,7 @@
     'Configuration',
     'ProxyConfiguration',
     'ProxyInfo',
+    'Request',
     'Webhook',
     'WebhookEventType',
     '__version__',
diff --git a/src/apify/apify_storage_client/_request_queue_client.py b/src/apify/apify_storage_client/_request_queue_client.py
@@ -2,6 +2,7 @@
 
 from typing import TYPE_CHECKING
 
+from more_itertools import chunked
 from typing_extensions import override
 
 from crawlee import Request
@@ -157,8 +158,11 @@ async def batch_add_requests(
         *,
         forefront: bool = False,
     ) -> BatchRequestsOperationResponse:
-        return BatchRequestsOperationResponse.model_validate(
-            await self._client.batch_add_requests(
+        processed = []
+        unprocessed = []
+
+        for chunk in chunked(requests, 25):  # The API endpoint won't accept more than 25 requests at once
+            response = await self._client.batch_add_requests(
                 requests=[
                     r.model_dump(
                         by_alias=True,
@@ -170,10 +174,18 @@ async def batch_add_requests(
                             'data',
                         },
                     )
-                    for r in requests
+                    for r in chunk
                 ],
                 forefront=forefront,
             )
+            processed.extend(response['processedRequests'])
+            unprocessed.extend(response['unprocessedRequests'])
+
+        return BatchRequestsOperationResponse.model_validate(
+            {
+                'processedRequests': processed,
+                'unprocessedRequests': unprocessed,
+            }
         )
 
     @override
diff --git a/src/apify/scrapy/middlewares/apify_proxy.py b/src/apify/scrapy/middlewares/apify_proxy.py
@@ -93,7 +93,7 @@ def process_exception(
         request: Request,
         exception: Exception,
         spider: Spider,
-    ) -> None | Request:
+    ) -> None:
         """Process an exception that occurs during request processing.
 
         Args:
@@ -102,8 +102,9 @@ def process_exception(
             spider: Scrapy Spider object.
 
         Returns:
-            If a TunnelError occurs, return the request object to halt its processing in the middleware pipeline.
-            Return None otherwise to allow the continuation of request processing.
+            Returning None, meaning Scrapy will continue processing this exception, executing any other
+            process_exception() methods of installed middleware, until no middleware is left and the default
+            exception handling kicks in.
         """
         Actor.log.debug(
             f'ApifyHttpProxyMiddleware.process_exception: request={request}, exception={exception}, spider={spider}',
@@ -114,9 +115,6 @@ def process_exception(
                 f'ApifyHttpProxyMiddleware: TunnelError occurred for request="{request}", '
                 'reason="{exception}", skipping...'
             )
-            return request
-
-        return None
 
     async def _get_new_proxy_url(self: ApifyHttpProxyMiddleware) -> ParseResult:
         """Get a new proxy URL.
diff --git a/tests/integration/test_request_queue.py b/tests/integration/test_request_queue.py
@@ -39,3 +39,68 @@ async def main() -> None:
         run_result = await actor.call()
         assert run_result is not None
         assert run_result['status'] == 'SUCCEEDED'
+
+    async def test_batch(self, make_actor: ActorFactory) -> None:
+        async def main() -> None:
+            async with Actor:
+                desired_request_count = 100
+                print('Opening request queue...')
+                # I have seen it get stuck on this call
+                rq = await Actor.open_request_queue()
+                # Add some requests
+                await rq.add_requests_batched([f'https://example.com/{i}' for i in range(desired_request_count)])
+
+                handled_request_count = 0
+                while next_request := await rq.fetch_next_request():
+                    print('Fetching next request...')
+                    queue_operation_info = await rq.mark_request_as_handled(next_request)
+                    assert queue_operation_info is not None
+                    assert queue_operation_info.was_already_handled is False
+                    handled_request_count += 1
+
+                assert handled_request_count == desired_request_count
+                print('Waiting for queue to be finished...')
+                is_finished = await rq.is_finished()
+                assert is_finished is True
+
+        actor = await make_actor('rq-batch-test', main_func=main)
+
+        run_result = await actor.call()
+        assert run_result is not None
+        assert run_result['status'] == 'SUCCEEDED'
+
+    async def test_batch_non_unique(self, make_actor: ActorFactory) -> None:
+        async def main() -> None:
+            from crawlee import Request
+
+            async with Actor:
+                desired_request_count = 100
+                print('Opening request queue...')
+                # I have seen it get stuck on this call
+                rq = await Actor.open_request_queue()
+                # Add some requests
+                await rq.add_requests_batched(
+                    [
+                        Request.from_url(f'https://example.com/{i}', unique_key=str(i - 1 if i % 4 == 1 else i))
+                        for i in range(desired_request_count)
+                    ]
+                )
+
+                handled_request_count = 0
+                while next_request := await rq.fetch_next_request():
+                    print('Fetching next request...')
+                    queue_operation_info = await rq.mark_request_as_handled(next_request)
+                    assert queue_operation_info is not None
+                    assert queue_operation_info.was_already_handled is False
+                    handled_request_count += 1
+
+                assert handled_request_count == desired_request_count * 3 / 4
+                print('Waiting for queue to be finished...')
+                is_finished = await rq.is_finished()
+                assert is_finished is True
+
+        actor = await make_actor('rq-batch-test', main_func=main)
+
+        run_result = await actor.call()
+        assert run_result is not None
+        assert run_result['status'] == 'SUCCEEDED'
diff --git a/tests/unit/scrapy/middlewares/test_apify_proxy.py b/tests/unit/scrapy/middlewares/test_apify_proxy.py
@@ -129,24 +129,14 @@ async def mock_get_new_proxy_url() -> ParseResult:
 
 
 @pytest.mark.parametrize(
-    ('exception', 'none_returned_values_is_expected'),
-    [
-        (TunnelError(), False),
-        (ValueError(), True),
-    ],
+    'exception',
+    [TunnelError(), ValueError()],
 )
 def test__process_exception(
     middleware: ApifyHttpProxyMiddleware,
     spider: DummySpider,
     dummy_request: Request,
     exception: Exception,
-    *,
-    none_returned_values_is_expected: bool,
 ) -> None:
-    returned_value = middleware.process_exception(dummy_request, exception, spider)
-
-    if none_returned_values_is_expected:
-        assert returned_value is None
-
-    else:
-        assert returned_value == dummy_request
+    returned_value = middleware.process_exception(dummy_request, exception, spider)  # type: ignore
+    assert returned_value is None
diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js
@@ -23,6 +23,7 @@ module.exports = {
         [
             '@apify/docs-theme',
             {
+                changelogFromRoot: true,
                 subNavbar: {
                     title: 'SDK for Python',
                     items: [
diff --git a/website/package-lock.json b/website/package-lock.json
diff --git a/website/package.json b/website/package.json
