add member spider

Author: oldshensheep

requirements-analysis.txt

@@ -1,2 +1 @@
 pandas==2.0.1
-plotly==5.14.1

v2ex_scrapy/pipelines.py

@@ -6,14 +6,13 @@
 
 # useful for handling different item types with a single interface
 
-from typing import Any, Union
+from typing import Any
 
 # don't remove
-import v2ex_scrapy.insert_ignore
 from v2ex_scrapy.DB import DB
 from v2ex_scrapy.items import CommentItem, MemberItem, TopicItem, TopicSupplementItem
 
-ItemsType = Union[TopicItem, CommentItem, MemberItem, TopicSupplementItem]
+ItemsType = TopicItem | CommentItem | MemberItem | TopicSupplementItem
 
 
 class TutorialScrapyPipeline:
@@ -31,22 +30,40 @@ def __init__(self):
 
     def process_item(
         self,
-        item: Union[ItemsType, Any],
+        item: ItemsType | Any,
         spider,
     ):
         if isinstance(item, (TopicItem, CommentItem, MemberItem, TopicSupplementItem)):
             item_type = type(item)
             self.data[item_type].append(item)
             if len(self.data[item_type]) >= self.BATCH:
-                self.db.session.add_all(self.data[item_type])
+                self.process_it(self.data[item_type])
                 self.data[item_type] = []
-                self.db.session.commit()
         return item
 
+    def process_it(self, items: list[ItemsType]):
+        if len(items) > 0 and isinstance(items[0], MemberItem):
+            self.process_members(items)
+        else:
+            self.db.session.add_all(items)
+            self.db.session.commit()
+
+    def process_members(self, items: list[MemberItem]):
+        for item in items:
+            e = (
+                self.db.session.query(MemberItem)
+                .where(MemberItem.username == item.username)
+                .first()
+            )
+            if e is None:
+                self.db.session.add_all([item])
+            elif e.uid is None:
+                e.uid = item.uid
+        self.db.session.commit()
+
     def save_all(self):
         for _, v in self.data.items():
-            self.db.session.add_all(v)
-            self.db.session.commit()
+            self.process_it(v)
 
     def close_spider(self, spider):
         self.save_all()

v2ex_scrapy/spiders/V2exMemberSpider.py

@@ -0,0 +1,41 @@
+import scrapy
+import scrapy.http.response.html
+
+from v2ex_scrapy import v2ex_parser
+from v2ex_scrapy.DB import DB
+from v2ex_scrapy.items import MemberItem
+
+
+class V2exTopicSpider(scrapy.Spider):
+    name = "v2ex-member"
+
+    def __init__(self, name=None, **kwargs):
+        super().__init__(name, **kwargs)
+        self.db = DB()
+        self.start_id = 1
+        self.end_id = 635000
+        self.logger.info(f"start from topic id {self.start_id}, end at {self.end_id}")
+
+    def start_requests(self):
+        for i in range(self.start_id, self.end_id + 1):
+            if not self.db.exist(MemberItem, i):
+                yield scrapy.Request(
+                    url=f"https://www.v2ex.com/uid/{i}",
+                    callback=self.parse,
+                    errback=self.member_err,
+                    cb_kwargs={"uid": i},
+                )
+
+    def parse(self, response: scrapy.http.response.html.HtmlResponse, uid: int):
+        for i in v2ex_parser.parse_member(response):
+            i.uid = uid
+            yield i
+
+    def member_err(self, failure):
+        yield MemberItem(
+            username="",
+            avatar_url="",
+            create_at=0,
+            social_link=[],
+            uid=failure.request.cb_kwargs["uid"],
+        )