♻️ Refactor FastCrawler core codebase and synced to FastAPI backend client (#53)

* 🐛 Fix: Controller injection to spiders from app

* 🐛 Fix Rocketry proto impl

* ♻️ Refactor interface to use list instead of set

* 🎨 Refactor design of engine

* ✅ Update new engine designs

* 🎨 Refactor spider w.r.t new engine design

* ✨ Add batching requests

* ✅ Add tests for batching

* ✅ Refactor existing test designs

* ✨ Add max_depth to control recursion

* ✅ Add tests for max_depth

* ✨ Add Sleep interval between request (Optional)

* ✅ Add test for interval between request

* ✨ Add cycle_sleep_interval & max_request_count

* ⚰️ Remove old aio engine design

* ♻️ Refactor AioEngine to send data or json base on contents

* ✅ Refactor aio engine tests for con limit

* 🐛 Fix await a Coroutine

* 🐛 Fix use ssl instead verify_ssl key in aiohttp

* ✅ Update test approx time to pass

* 🐛 Fix list of tasks return in the `RocketryApplication`

* ⚡️ Replace if statement by an expression

* 🚑 Fix toggle task new_status

* ⚡️ Change task schedule returns task instead of None

* ♻️ Refactor core's structure for typing improvements

* ♻️ Refactor task manager structure for typing improvements

* ♻️ Refactor engine to use dataclass and have better typings

* 🐛 Fix minor issue with task registry

* ♻️ Refactor core with new design

* 🏷️  Improve explicit typing for URL

* ♻️ Refactor tests to pass with new design

* 🎨 Add customized exception for bad configuration

* ♻️ Rename Proto to ABC

* ♻️ Refactor task to be immutable

* ♻️ Refactor class variable names to be PEP-8 friendly

---------

Co-authored-by: Sadegh Yazdani <[email protected]>

Author: ManiMozaffar

docs_src/processor/tutorial001.py

@@ -33,7 +33,7 @@ async def main():
         cond="every 1 second",
         controller=ProcessController(app=RocketryApplication()),
     )
-    await process.add_spiders()
+    await process.add_spiders_to_controller()
     assert len(await process.controller.app.get_all_tasks()) == 1
     await process.start(silent=False)
 

fastcrawler/core/app.py

@@ -1,13 +1,20 @@
 import asyncio
-from typing import Callable
+from typing import Any, Coroutine
 
-from fastcrawler.exceptions import NoCrawlerFoundError
+from fastcrawler.exceptions import NoCrawlerFoundErrorError
 from fastcrawler.schedule.adopter import ProcessController, RocketryApplication
-from fastcrawler.schedule.contracts import ControllerProto
+from fastcrawler.schedule.contracts import ControllerABC
 
 from .process import Process
 
 
+def list_process(crawlers: list[Process] | Process) -> list[Process]:
+    if isinstance(crawlers, Process):
+        return [crawlers]
+    else:
+        return crawlers
+
+
 class FastCrawler:
     """The client interface to start all crawlers.
     Initialize all crawlers
@@ -22,47 +29,48 @@ class FastCrawler:
 
     """
 
-    controller: ControllerProto | None = None
+    controller: ControllerABC | None = None
 
     def __init__(
         self,
         crawlers: list[Process] | Process,
-        controller: ControllerProto | None = None,
+        controller: ControllerABC | None = None,
     ):
         """Initialize FastCrawler with defined crawlers"""
-        ...
-        if isinstance(crawlers, Process):
-            self.crawlers = [
-                crawlers,
-            ]
-        else:
-            self.crawlers = crawlers
-
+        self.crawlers = list_process(crawlers)
         self.controller = controller or ProcessController(app=RocketryApplication())
         if not self.crawlers or len(self.crawlers) == 0:
-            raise NoCrawlerFoundError
+            raise NoCrawlerFoundErrorError
 
     @property
-    def get_all_serves(self) -> list[Callable]:
+    def get_all_serves(self) -> list[Coroutine[Any, Any, None]]:
         """get all application to be served"""
+        assert self.controller is not None
         return [
             self.controller.app.serve(),
         ]
 
     async def serve(self) -> None:
-        """Serve protocol for uvicorn"""
+        """
+        Serve protocol for uvicorn, useful with combination
+            with other tools if get_all_serves is customized
+        """
         await asyncio.gather(*self.get_all_serves)
         return None
 
     async def start(self, silent=True) -> None:
         """Start all crawlers in background explicitly without schedule"""
+
+        # TODO: make here multi processing, for more than one process!
+        # or use rocketry to trigger the tasks, if possible :)
         await asyncio.gather(*[crawler.start(silent) for crawler in self.crawlers])
         return None
 
     async def run(self) -> None:
         """Run all crawlers in background explicitly with schedule"""
         for crawler in self.crawlers:
-            await crawler.add_spiders()
+            crawler.controller = self.controller
+            await crawler.add_spiders_to_controller()
         await self.serve()
         return None
 
@@ -76,6 +84,7 @@ async def shutdown(self) -> None:
 
     async def _shutdown(self) -> None:
         """Safe shut down event for application crawler"""
+        assert self.controller is not None
         await self.shutdown()
         await self.controller.shut_down()
         return None

fastcrawler/core/process.py

@@ -1,7 +1,7 @@
 from uuid import uuid4
 
 from fastcrawler.core.spider import Spider
-from fastcrawler.schedule.contracts import ControllerProto
+from fastcrawler.schedule.contracts import ControllerABC
 from fastcrawler.schedule.schema import Task
 
 
@@ -14,7 +14,7 @@ class Process:
     def __init__(
         self,
         spider: Spider,
-        controller: ControllerProto | None = None,
+        controller: ControllerABC | None = None,
         cond: str | Task | None = None,
         *args,
         **kwargs,
@@ -23,15 +23,15 @@ def __init__(
 
         Args:
             spider (Spider): _description_
-            controller (None | ControllerProto, optional): _description_. Defaults to None.
+            controller (None | ControllerABC, optional): _description_. Defaults to None.
             cond (Task | None, optional): _description_. Defaults to None.
         """
         if isinstance(cond, Task):
             self.task = cond
         else:
             self.task = Task(
                 start_cond=cond or "every 1 second",
-                name=spider.__class__.__name__ + str(uuid4()),
+                name=f"{uuid4()}@{spider.__class__.__name__}",
             )
         self.args = args
         self.kwargs = kwargs
@@ -44,23 +44,26 @@ async def start(self, silent: bool = True) -> None:
         This method will disable scheduler temporarily to avoid duplicate running
         """
         if self.controller:
-            await self.controller.toggle_task(self.task.name, new_status=False)
+            await self.controller.toggle_task(str(self.task.name), new_status=False)
         await self.spider.start(silent=silent)
         if self.controller:
-            await self.controller.toggle_task(self.task.name, new_status=True)
+            await self.controller.toggle_task(str(self.task.name), new_status=True)
         return None
 
     async def stop(self) -> None:
         """Stop the crawling process"""
-        self.spider.is_stopped = True
+        for instance in self.spider.instances:
+            instance.is_stopped = True
+
         if self.controller:
-            self.controller.toggle_task(self.task.name, new_status=False)
+            await self.controller.toggle_task(str(self.task.name), new_status=False)
         return None
 
-    async def add_spiders(self) -> None:
+    async def add_spiders_to_controller(self) -> None:
         """
         Run the crawling process
         """
+        assert self.controller is not None
         if self.task:
             await self.controller.add_task(self.spider.start, self.task)
         else: