✨ Add Core (#40)

* 💩 Repeat DI injection code

* ✏️ fix component integration bugs with core

* ✨ implement Spider

* ⚡️ Improve Spider's perfomance

* 💩 add shit doc src samples

* 💩 implement base code for core

* ♻️ adapted design to core

* ➕ updated dependency

* ✅ refactor old tests to new design

* 💩 add base benchmark

* ✏️ Fix rocketry adaptor starting issue

* 🐛 Fix: Crawler starting/shut down issues

* ⚰️ Remove benchmarks

* 🎨 Improve schedule design to adopt core

* 📝 Add new sample doc src for processor

* 🎨 Fix imports w.t.r new design

* ♻️ Refactor Engine to adopt to core design

* ♻️ Refactor DI injection to adopt to core design

* ⚡️ Add Spider/Processor to Core

* ♻️ Refactor Core's Spider to improve design

* ♻️ Refactor Old test to pass w.t.r new designs

* ✅ Add test for core's component

* 🔥 Remove extra fastapi folders

* 🔥 Remove playwright proto

* 🐛 Fix Aio HTTP design

* 🎨 Improve typing/doc strings

* ✏️ Fix doc and code typo

---------

Co-authored-by: Sadegh Yazdani

Author: ManiMozaffar

.github/ISSUE_TEMPLATE/PULL_request_template.md

@@ -2,7 +2,7 @@
 🚨Please review the [guidelines for contributing](../CONTRIBUTING.md) to this repository.
 
 - [ ] Make sure you are requesting to **pull a topic/feature/bugfix branch** (right side). Don't request your master!
-- [ ] Make sure you have 100% test coverage 
+- [ ] Make sure you have 100% test coverage
 - [ ] Check the commit's or even all commits' message styles matches our requested structure.
 - [ ] Check your code linting
 

.github/ISSUE_TEMPLATE/bug.yml

@@ -44,4 +44,4 @@ body:
       value: |-
         ## Thanks 🙏
     validations:
-      required: false
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -2,4 +2,4 @@ blank_issues_enabled: false
 contact_links:
   - name: Question or Problem
     about: Ask a question or ask about a problem in GitHub Discussions.
-    url: https://t.me/fastcrawler
+    url: https://t.me/fastcrawler

.pre-commit-config.yaml

@@ -14,12 +14,12 @@ repos:
     rev: 6.0.0
     hooks:
       - id: flake8
-  - repo: 'https://github.com/pre-commit/mirrors-mypy'
-    rev: v1.4.1
-    hooks:
-      - id: mypy
-        name: mypy (fastcrawler)
-        files: ^fastcrawler/
+  # - repo: 'https://github.com/pre-commit/mirrors-mypy'
+  #   rev: v1.4.1
+  #   hooks:
+  #     - id: mypy
+  #       name: mypy (fastcrawler)
+  #       files: ^fastcrawler/
       # - id: mypy
       #   name: mypy (test)
       #   files: ^test/

docs_src/initilizing_project/sample1/wikipedia.py

@@ -1,7 +1,7 @@
 # pylint: disable-all
 
 
-from fastcrawler import BaseModel, Crawler, CSSField, Spider, XPATHField
+from fastcrawler import BaseModel, CSSField, Process, Spider, XPATHField
 from fastcrawler.engine import AioHttpEngine
 
 
@@ -36,4 +36,4 @@ async def save_data(self, data: ArticleData):
         ...  # save parsed data to database
 
 
-wiki_spider = Crawler(WikiArticleFinder >> WikiArticleRetirever)
+wiki_spider = Process(WikiArticleFinder >> WikiArticleRetirever)

docs_src/initilizing_project/sample2/imdb.py

@@ -0,0 +1,36 @@
+# pylint: disable-all
+
+
+from fastcrawler import BaseModel, Crawler, CSSField, Spider, XPATHField
+from fastcrawler.engine import AioHttpEngine
+
+
+class PageResolver(BaseModel):
+    class Config:
+        url_resolver = XPATHField("//a[contains(@href, 'en.Imdbpedia')]/@href")
+
+
+class ArticleData(BaseModel):
+    title: str = CSSField("h1.firstHeading", extract="text")  # gets text
+    body: str = CSSField("div.mw-body-content > div.mw-parser-output")  # gets inner HTML
+
+
+class ImdbBaseSpider(Spider):
+    engine = AioHttpEngine
+    concurrency = 100
+
+
+class ImdbArticleFinder(ImdbBaseSpider):
+    data_model = PageResolver
+    req_count = 1_000_000
+    start_url = [
+        "https://meta.Imdbmedia.org/Imdb/List_of_Imdbpedias",
+    ]
+
+
+class ImdbArticleRetirever(ImdbBaseSpider):
+    data_model = ArticleData
+    req_count = 1_000_000
+
+    async def save_data(self, data: ArticleData):
+        ...  # save parsed data to database

docs_src/initilizing_project/sample2/main.py

@@ -0,0 +1,10 @@
+from fastcrawler import FastCrawler, Process
+
+from .imdb import ImdbArticleFinder, ImdbArticleRetirever
+from .wikipedia import WikiArticleFinder, WikiArticleRetirever
+
+app = FastCrawler(
+    Process(ImdbArticleFinder >> ImdbArticleRetirever, cond="every 3 minute"),
+    Process(WikiArticleFinder >> WikiArticleRetirever, cond="every 3 minute"),
+)
+app.run()

docs_src/initilizing_project/sample2/wikipedia.py

@@ -0,0 +1,39 @@
+# pylint: disable-all
+
+
+from fastcrawler import BaseModel, CSSField, Process, Spider, XPATHField
+from fastcrawler.engine import AioHttpEngine
+
+
+class PageResolver(BaseModel):
+    class Config:
+        url_resolver = XPATHField("//a[contains(@href, 'en.wikipedia')]/@href")
+
+
+class ArticleData(BaseModel):
+    title: str = CSSField("h1.firstHeading", extract="text")  # gets text
+    body: str = CSSField("div.mw-body-content > div.mw-parser-output")  # gets inner HTML
+
+
+class WikiBaseSpider(Spider):
+    engine = AioHttpEngine
+    concurrency = 100
+
+
+class WikiArticleFinder(WikiBaseSpider):
+    data_model = PageResolver
+    req_count = 1_000_000
+    start_url = [
+        "https://meta.wikimedia.org/wiki/List_of_Wikipedias",
+    ]
+
+
+class WikiArticleRetirever(WikiBaseSpider):
+    data_model = ArticleData
+    req_count = 1_000_000
+
+    async def save_data(self, data: ArticleData):
+        ...  # save parsed data to database
+
+
+wiki_spider = Process(WikiArticleFinder >> WikiArticleRetirever)

docs_src/processor/tutorial001.py

@@ -0,0 +1,41 @@
+import asyncio
+
+from fastcrawler import BaseModel, Depends, Process, Spider, XPATHField
+from fastcrawler.schedule import ProcessController, RocketryApplication
+
+
+class PersonData(BaseModel):
+    name: str = XPATHField(query="//td[1]", extract="text")
+    age: int = XPATHField(query="//td[2]", extract="text")
+
+
+class PersonPage(BaseModel):
+    person: list[PersonData] = XPATHField(query="//table//tr", many=True)
+
+
+async def get_urls():
+    return {f"http://localhost:8000/persons/{id}" for id in range(20)}
+
+
+class MySpider(Spider):
+    engine_request_limit = 20
+    data_model = PersonPage
+    start_url = Depends(get_urls)
+
+    async def save(self, all_data: list[PersonPage]):
+        assert all_data is not None
+        assert len(all_data) == 20
+
+
+async def main():
+    process = Process(
+        spider=MySpider(),
+        cond="every 1 second",
+        controller=ProcessController(app=RocketryApplication()),
+    )
+    await process.add_spiders()
+    assert len(await process.controller.app.get_all_tasks()) == 1
+    await process.start(silent=False)
+
+
+asyncio.run(main())

fastcrawler/__init__.py

@@ -1,7 +1,7 @@
-from .core import Crawler, FastCrawler, Spider
+from .core import FastCrawler, Process, Spider
 from .engine import AioHttpEngine
 from .parsers import BaseModel, CSSField, RegexField, XPATHField
-from .schedule import RocketryApplication, RocketryController
+from .schedule import ProcessController, RocketryApplication
 from .utils import Depends
 
 __all__ = [
@@ -11,9 +11,9 @@
     "RegexField",
     "Depends",
     "Spider",
-    "Crawler",
+    "Process",
     "FastCrawler",
     "RocketryApplication",
-    "RocketryController",
+    "ProcessController",
     "AioHttpEngine",
 ]