Let provider params influece AnyResponse resolution (#200)

Author: Gallaecio

scrapy_zyte_api/providers.py

@@ -174,8 +174,10 @@ async def __call__(  # noqa: C901
 
         if AnyResponse in to_provide:
             if (
-                item_requested and extract_from != "httpResponseBody"
-            ) or extract_from == "browserHtml":
+                (item_requested and extract_from != "httpResponseBody")
+                or extract_from == "browserHtml"
+                or zyte_api_meta.get("browserHtml", False) is True
+            ):
                 html_requested = True
             elif extract_from == "httpResponseBody" or http_response_needed:
                 zyte_api_meta["httpResponseBody"] = True

tests/test_providers.py

@@ -439,6 +439,61 @@ def parse_(self, response: DummyResponse, page: SomePage):
     assert type(item["page"].response.response) is HttpResponse
 
 
+@ensureDeferred
+async def test_provider_any_response_http_response_param(mockserver):
+    @attrs.define
+    class SomePage(BasePage):
+        response: AnyResponse
+
+    class ZyteAPISpider(Spider):
+        def start_requests(self):
+            yield Request(self.url, callback=self.parse_)
+
+        def parse_(self, response: DummyResponse, page: SomePage):
+            yield {"page": page}
+
+    settings = provider_settings(mockserver)
+    settings["ZYTE_API_PROVIDER_PARAMS"] = {"httpResponseBody": True}
+    item, url, crawler = await crawl_single_item(ZyteAPISpider, HtmlResource, settings)
+    params = crawler.engine.downloader.handlers._handlers["http"].params
+
+    assert len(params) == 1
+    assert params[0] == {
+        "url": url,
+        "httpResponseBody": True,
+        "httpResponseHeaders": True,
+    }
+    assert type(item["page"].response) is AnyResponse
+    assert type(item["page"].response.response) is HttpResponse
+
+
+@ensureDeferred
+async def test_provider_any_response_browser_html_param(mockserver):
+    @attrs.define
+    class SomePage(BasePage):
+        response: AnyResponse
+
+    class ZyteAPISpider(Spider):
+        def start_requests(self):
+            yield Request(self.url, callback=self.parse_)
+
+        def parse_(self, response: DummyResponse, page: SomePage):
+            yield {"page": page}
+
+    settings = provider_settings(mockserver)
+    settings["ZYTE_API_PROVIDER_PARAMS"] = {"browserHtml": True}
+    item, url, crawler = await crawl_single_item(ZyteAPISpider, HtmlResource, settings)
+    params = crawler.engine.downloader.handlers._handlers["http"].params
+
+    assert len(params) == 1
+    assert params[0] == {
+        "url": url,
+        "browserHtml": True,
+    }
+    assert type(item["page"].response) is AnyResponse
+    assert type(item["page"].response.response) is BrowserResponse
+
+
 @ensureDeferred
 async def test_provider_any_response_product(mockserver):
     @attrs.define