yes

Author: oldshensheep

v2ex_scrapy/spiders/V2exMemberSpider.py

@@ -1,5 +1,6 @@
 import scrapy
 import scrapy.http.response.html
+from scrapy.spidermiddlewares.httperror import HttpError
 
 from v2ex_scrapy import v2ex_parser
 from v2ex_scrapy.DB import DB
@@ -9,11 +10,11 @@
 class V2exTopicSpider(scrapy.Spider):
     name = "v2ex-member"
 
-    def __init__(self, name=None, **kwargs):
-        super().__init__(name, **kwargs)
+    def __init__(self, start_id=1, end_id=635000, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self.db = DB()
-        self.start_id = 1
-        self.end_id = 635000
+        self.start_id = start_id
+        self.end_id = end_id
         self.logger.info(f"start from topic id {self.start_id}, end at {self.end_id}")
 
     def start_requests(self):
@@ -25,17 +26,20 @@ def start_requests(self):
                     errback=self.member_err,
                     cb_kwargs={"uid": i},
                 )
+            else:
+                self.logger.info(f"skip member id:{i}, because it exists")
 
     def parse(self, response: scrapy.http.response.html.HtmlResponse, uid: int):
         for i in v2ex_parser.parse_member(response):
             i.uid = uid
             yield i
 
     def member_err(self, failure):
-        yield MemberItem(
-            username="",
-            avatar_url="",
-            create_at=0,
-            social_link=[],
-            uid=failure.request.cb_kwargs["uid"],
-        )
+        if failure.check(HttpError):
+            yield MemberItem(
+                username="",
+                avatar_url="",
+                create_at=0,
+                social_link=[],
+                uid=failure.request.cb_kwargs["uid"],
+            )

v2ex_scrapy/spiders/V2exNodeTopicSpider.py

@@ -10,7 +10,7 @@
 from v2ex_scrapy import utils
 
 
-class V2exTopicSpider(scrapy.Spider):
+class V2exNodeTopicSpider(scrapy.Spider):
     name = "v2ex-node"
 
     UPDATE_TOPIC_WHEN_REPLY_CHANGE = True

v2ex_scrapy/spiders/V2exSpider.py

@@ -6,13 +6,13 @@
 from v2ex_scrapy.spiders.CommonSpider import CommonSpider
 
 
-class V2exTopicSpider(scrapy.Spider):
+class V2exSpider(scrapy.Spider):
     name = "v2ex"
     FORCE_UPDATE_TOPIC = False
     UPDATE_COMMENT = True
 
-    def __init__(self, name=None, **kwargs):
-        super().__init__(name, **kwargs)
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self.db = DB()
         self.start_id = 1
         self.end_id = 1000000