✨ Feature/scheduler (#5)

* feat: add basic of rocketry

* feat: add rocketry

* feat: pycharm conf

* removed: CI pipeline(temp)

* fixed: some typos

* added: interface for task manager

* removed: fastapi

* added: Task prototype

* implemented: rocketry proto

* ✏️ fix proto/typo/lint

* ✅ Add tests for schedule

* test: add new tests for `RocketryApplication`, `RocketryManager`
* fix: some problem for default value in schema
* ci: add rocketry, redbird from Mani repo
* ci: update pydantic to 2.0.2
* build: update setup.cfg due to #9

* 🐛 fix selectors base

* ✏️ improve names & code quality

* ✅ update: test code quality

---------

Co-authored-by: ManiMozaffar <[email protected]>
Co-authored-by: Sadegh Yazdani

Author: vahidtwo

.gitignore

@@ -163,4 +163,4 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/

checks.sh

@@ -26,6 +26,24 @@ def __init__(self, model):
         super().__init__(self.message)
 
 
+class TaskNotFound(BaseModelError):
+    def __init__(self, task_name):
+        super().__init__(
+            f"The Task with name={task_name} has not been found",
+            "\nPlease check your input and be sure the task name is correct",
+        )
+
+
+class NoCrawlerFound(BaseModelError):
+    def __init__(self):
+        super().__init__(
+            "No task has been registered in the application."
+            "\nPlease make sure that you've assigned the crawlers to the application"
+            "so the application is aware of the crawlers."
+            "\nThis may also raise if you have overridden the library's startup built in method"
+        )
+
+
 class ProcessorNotSupported(BaseModelError):
     def __init__(self, model):
         self.model = model

fastcrawler/exceptions.py

@@ -6,11 +6,9 @@
 class ParserProtocol(Protocol):
     def __init__(self, scraped_data: Any):
         """Initilize the parser with the given data (html/json/etc)"""
-        ...
 
     def parse(self, model: Any) -> Any:
         """
         Parse the saved data, with given model, which should be a pydantic model
             imported from fastcrawler library
         """
-        ...

fastcrawler/parsers/base.py

@@ -36,7 +36,7 @@ def __init__(
 
     def __repr__(self):
         """Represents a selector for debugging purposes"""
-        return f"Field(type={self.__class__.__name__} extract={self.extract}, " f"many={self.many}, query={self.query})"
+        return f"Field(type={self.__class__.__name__} extract={self.extract}, many={self.many}, query={self.query})"
 
     def resolve(self, scraped_data, model):
         """Must be implemented by outer classes.

fastcrawler/parsers/selectors/base.py

@@ -7,9 +7,10 @@
 from fastcrawler.parsers.utils import _UNSET
 
 from ..processors.base import ProcessorInterface
+from .base import BaseSelector
 
 
-class _CSSField:
+class _CSSField(BaseSelector):
     """
     CSSSelectorField represents a field that can be retrieved from a given HTML
         document using CSS selectors.

fastcrawler/parsers/selectors/css.py

@@ -6,8 +6,10 @@
 from fastcrawler.parsers.pydantic import BaseModelType
 from fastcrawler.parsers.utils import _UNSET
 
+from .base import BaseSelector
 
-class _RegexField:
+
+class _RegexField(BaseSelector):
     """
     RegexField represents a field that can be retrieved from a given HTML
     document using Regex.

fastcrawler/parsers/selectors/regex.py

@@ -7,9 +7,10 @@
 from fastcrawler.parsers.utils import _UNSET
 
 from ..processors.base import ProcessorInterface
+from .base import BaseSelector
 
 
-class _XPATHField:
+class _XPATHField(BaseSelector):
     """
     XPATHField represents a field that can be retrieved from a given HTML
     document using XPath.

fastcrawler/parsers/selectors/xpath.py

@@ -0,0 +1,90 @@
+from typing import Callable
+
+from rocketry import Rocketry
+from rocketry.conditions.api import cron
+
+from fastcrawler.exceptions import TaskNotFound
+
+from .schema import Task
+
+
+class RocketryApplication:
+    def __init__(self, *args, **kwargs):
+        self.task_lib: Rocketry = Rocketry(*args, **kwargs)
+
+    async def serve(self, *args, **kwargs):  # pragma: no cover
+        """Proto to serve with uvicorn"""
+        await self.start_up()
+        return await self.task_lib.serve(*args, **kwargs)
+
+    async def get_all_tasks(self) -> set[Task]:
+        return self.task_lib.session.tasks
+
+    async def add_task(self, task_func: Callable, settings: Task) -> None:
+        """
+        ...
+        """
+        self.task_lib.task(**dict(settings))(task_func)
+        return None
+
+    async def start_up(self) -> None:
+        """
+        Run Startup Event
+        """
+
+    async def shut_down(self) -> None:
+        self.task_lib.session.shut_down()
+        return None
+
+
+class RocketryManager:
+    def __init__(self, app: RocketryApplication):
+        self.app = app
+
+    async def all(self) -> set[Task]:
+        """
+        Return all tasks from internal
+        """
+        return await self.app.get_all_tasks()
+
+    async def add_task(self, task_func: Callable, settings: Task) -> None:
+        """
+        Add tasks within internal python API
+        """
+        await self.app.add_task(task_func, settings)
+        return None
+
+    async def change_task_schedule(
+        self,
+        task_name: str,
+        schedule: str,
+    ) -> None:
+        """
+        Reschedule a task
+            schedule:
+                - can be string
+                    `every 2 seconds`
+                - can be cron
+                    `*/2 * * * *`
+        """
+        for task in await self.app.get_all_tasks():
+            if task.name == task_name:
+                if schedule.count(" ") == 4:
+                    task.start_cond = cron(schedule)
+                else:
+                    task.start_cond = schedule
+                return None
+        raise TaskNotFound(task_name)
+
+    async def toggle_task(self, task_name: str) -> None:
+        """
+        Disables or enable one task
+        """
+        for task in await self.app.get_all_tasks():
+            if task.name == task_name:
+                if task.disabled:
+                    task.disabled = False
+                else:
+                    task.disabled = True
+                return None
+        raise TaskNotFound(task_name)

fastcrawler/schedule/adopter.py

@@ -0,0 +1,61 @@
+from typing import Callable, Protocol
+
+from .schema import Task
+
+
+class TaskApplicationProto(Protocol):  # pragma: no cover
+    task_lib: Callable
+
+    def __init__(self, *args, **kwargs):
+        """Initialize task application"""
+
+    async def serve(self, *args, **kwargs):
+        """Proto to serve with Uvicorn"""
+
+    async def get_all_tasks(self) -> set[Task]:
+        """Returns all tasks that exists in Fast Crawler"""
+
+    async def add_task(self, *args, **kwargs) -> None:
+        """Dynamically add a task to fast crawler"""
+
+    async def start_up(self) -> None:
+        """Manage start up actvity"""
+
+    async def shut_down(self) -> None:
+        """Manage shut down activity"""
+
+
+class TaskControllerProto(Protocol):  # pragma: no cover
+    app: TaskApplicationProto
+
+    def __init__(self, app: TaskApplicationProto):
+        """Initialize task application
+
+        Args:
+            app (TaskProcessor): _description_
+        """
+
+    async def all(self) -> list[Task]:
+        """
+        Return all tasks from internal
+        """
+
+    async def add_task(self, task_func: Callable, settings: Task) -> None:
+        """
+        Add tasks within internal python API
+        """
+
+    async def change_task_schedule(self, task_name: str, schedule: str) -> None:
+        """
+        Reschedule a task
+            schedule:
+                - can be string
+                    `every 2 seconds`
+                - can be cron
+                    `*/2 * * * *`
+        """
+
+    async def toggle_task(self, task_name: str) -> None:
+        """
+        Disables or enable one task
+        """