feat(spiders): Change items hook to make it for processing items + add a stat for this

Author: D4Vinci

scrapling/spiders/engine.py

@@ -102,12 +102,17 @@ async def _process_request(self, request: Request) -> None:
                         self.stats.offsite_requests_count += 1
                         log.debug(f"Filtered offsite request to: {result.url}")
                 elif isinstance(result, dict):
-                    self.stats.items_scraped += 1
-                    self._items.append(result)
-                    if self._item_stream:
-                        await self._item_stream.send(result)
-                    await self.spider.on_scraped_item(result)
-                    log.debug(f"Scraped from {str(response)}\n{result}")
+                    processed_result = await self.spider.on_scraped_item(result)
+                    if processed_result:
+                        self.stats.items_scraped += 1
+                        log.debug(f"Scraped from {str(response)}\n{processed_result}")
+                        if self._item_stream:
+                            await self._item_stream.send(processed_result)
+                        else:
+                            self._items.append(processed_result)
+                    else:
+                        self.stats.items_dropped += 1
+                        log.warning(f"Dropped from {str(response)}\n{processed_result}")
                 elif result is not None:
                     log.error(f"Spider must return Request, dict or None, got '{type(result)}' in {request}")
         except Exception as e:

scrapling/spiders/result.py

@@ -50,6 +50,7 @@ class CrawlStats:
     offsite_requests_count: int = 0
     response_bytes: int = 0
     items_scraped: int = 0
+    items_dropped: int = 0
     start_time: float = 0.0
     end_time: float = 0.0
     download_delay: float = 0.0
@@ -85,6 +86,7 @@ def increment_requests_count(self, sid: str) -> None:
     def to_dict(self) -> dict[str, Any]:
         return {
             "items_scraped": self.items_scraped,
+            "items_dropped": self.items_dropped,
             "elapsed_seconds": round(self.elapsed_seconds, 2),
             "download_delay": round(self.download_delay, 2),
             "concurrent_requests": self.concurrent_requests,

scrapling/spiders/spider.py

@@ -160,9 +160,9 @@ async def on_error(self, request: Request, error: Exception) -> None:
         """
         self.logger.error(error, exc_info=error)
 
-    async def on_scraped_item(self, item: dict[str, Any]) -> None:
-        """Handle a scraped item. Override or extend for item pipelines."""
-        pass
+    async def on_scraped_item(self, item: Dict[str, Any]) -> Dict[str, Any] | None:
+        """A hook to be overridden by users to do some processing on scraped items, return `None` to drop the item silently."""
+        return item
 
     async def is_blocked(self, response: "Response") -> bool:
         """Check if the response is blocked. Users should override this for custom detection logic."""