rename to v2ex_scrapy

Author: oldshensheep

README.md

@@ -0,0 +1,55 @@
+# 一个爬取v2ex.com网站的爬虫
+
+学习scrapy写的一个小爬虫
+
+## 不建议自行运行爬虫，数据已经有了
+
+## 爬取相关数据说明
+
+数据都放在了sqlite数据库，方便分，整个数据库大小2.1GB。
+
+爬虫源码放在了GitHub，在GitHub我release了完整的sqlite数据库文件
+
+爬虫从`topic_id = 1`开始爬，路径为`https://www.v2ex.com/t/{topic_id}`。 服务器可能返回404/403/302/200，如果是404说明帖子被删除了，如果是403说明是爬虫被限制了，302一般是跳转到登陆页面，有的也是跳转到主页，200返回正常页面。
+
+爬虫没有登陆，所以爬取的数据不完全，比如水下火热的帖子就没有爬到，还有就是如果是302的帖子会记录帖子id，404/403不会记录。
+
+爬取过程中会帖子内容，评论，以及评论的用户信息。
+
+注1：爬了一半才发现V站帖子附言没有爬，附言从`topic_id = 448936`才会爬取
+
+注2：select count(*) from member 得到的用户数比较小，大概20W，是因为爬取过程中是根据评论，以及发帖信息爬取用户的，如果一个用户注册之后既没有评论也没有发帖，那么这个账号就爬不到。还有就是因为部分帖子访问不了，也可能导致部分账号没有爬。还有部分用户号被删除，这一部分也没有爬。（代码改了，可以爬，但是都已经爬完了……）
+
+注3：时间均为UTC+0的秒数
+
+## 运行
+
+### 安装依赖
+
+```bash
+pip install -r .\requirements.txt
+```
+
+### 配置
+
+#### 代理
+
+更改 `v2ex_scrapy/settings.py` 中 `PROXIES`的值 如
+
+```python
+[
+     "http://127.0.0.1:7890"
+]
+```
+
+请求会随机选择一个代理，如果需要更高级的代理方式可以使用第三方库，或者自行实现Middleware
+
+### 运行爬虫
+
+```bash
+scrapy crawl v2ex
+```
+
+### 注意事项
+
+爬取过程中出现403基本上是因为IP被限制了，等待一段时间即可

analysis.py

@@ -0,0 +1,57 @@
+import plotly.express as px
+import plotly.graph_objects as go
+import pandas
+import sqlite3
+
+conn = sqlite3.connect("./v2ex.sqlite")
+c = conn.cursor()
+
+topic = pandas.read_sql(
+    """
+SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS topic_count
+FROM topic
+GROUP BY date;
+    """,
+    conn,
+)
+# drop 1970
+fig = px.line(
+    topic[1:],
+    x="date",
+    y="topic_count",
+)
+fig.show()
+
+comment = pandas.read_sql(
+    """
+SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS comment_count
+FROM comment
+GROUP BY date;
+    """,
+    conn,
+)
+fig = px.line(
+    comment,
+    x="date",
+    y="comment_count",
+)
+fig.show()
+
+user = pandas.read_sql(
+    """
+SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS user_count
+FROM member
+GROUP BY date;
+    """,
+    conn,
+)
+
+
+# drop 1970
+fig = px.line(
+    user[1:],
+    x="date",
+    y="user_count",
+)
+
+fig.show()

db.sql

@@ -0,0 +1,96 @@
+select count(*)
+from comment;
+-- where create_at between strftime('%s', '2013-01-01') and strftime('%s', '2014-12-31');
+
+select count(*)
+from member;
+
+select count(*)
+from topic;
+
+-- top comment by thank_count
+select topic_id, c.id, thank_count
+from comment c
+         left join topic t on t.id = c.topic_id
+order by thank_count desc;
+
+-- top topic by votes
+select id, title, votes
+from topic
+order by votes desc;
+
+-- top topic by clicks
+select id, title, clicks
+from topic
+order by clicks desc;
+
+
+-- top node
+select node, count(node) as count
+from topic
+group by node
+order by count desc;
+
+-- comment number group by user
+select commenter, count(commenter) as comment_count
+from comment
+group by commenter
+order by comment_count desc;
+
+-- topic number group by user
+select author, count(author) as topic_count
+from topic
+group by author
+order by topic_count desc;
+
+-- topic number group by year
+SELECT date,
+       SUM(topic_count) OVER (ORDER BY date ) AS cumulative_topic_count
+FROM (SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS topic_count
+      FROM topic
+      GROUP BY date)
+ORDER BY date;
+
+-- user number group by year
+SELECT date,
+       SUM(user_count) OVER (ORDER BY date ) AS cumulative_user_count
+FROM (SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS user_count
+      FROM member
+      GROUP BY date)
+ORDER BY date;
+
+-- comment number group by year
+SELECT date,
+       SUM(comment_count) OVER (ORDER BY date ) AS cumulative_comment_count
+FROM (SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS comment_count
+      FROM comment
+      GROUP BY date)
+ORDER BY date;
+
+-- new topic number group by year
+SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS topic_count
+FROM topic
+GROUP BY date;
+
+-- new user number group by year
+SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS user_count
+FROM member
+GROUP BY date;
+
+-- new comment number group by year
+SELECT strftime('%Y-%m', create_at, 'unixepoch') AS date, COUNT(*) AS comment_count
+FROM comment
+GROUP BY date;
+
+-- tag usage count
+select t.value as tag, count(*) as count
+from topic,
+     json_each(tag) as t
+group by t.value
+order by count desc;
+
+-- node usage count
+select node, count(*) as count
+from topic
+group by node
+order by count desc;

query.sql

@@ -0,0 +1,5 @@
+arrow==1.2.3
+pandas==2.0.1
+plotly==5.14.1
+Scrapy==2.9.0
+SQLAlchemy==2.0.17

requirements.txt

@@ -4,8 +4,8 @@
 # https://scrapyd.readthedocs.io/en/latest/deploy.html
 
 [settings]
-default = tutorial_scrapy.settings
+default = v2ex_scrapy.settings
 
 [deploy]
 #url = http://localhost:6800/
-project = tutorial_scrapy
+project = v2ex_scrapy

scrapy.cfg

@@ -1,17 +1,14 @@
 import json
-import sqlite3
-from typing import List, Type, Union
+from typing import Type, Union
 
-from sqlalchemy import create_engine, exists, func, select, text
+from sqlalchemy import create_engine, text
 from sqlalchemy.orm import Session
 
-from tutorial_scrapy import utils
-from tutorial_scrapy.items import (
+from v2ex_scrapy.items import (
     Base,
     CommentItem,
     MemberItem,
     TopicItem,
-    TopicSupplementItem,
 )
 
 
@@ -25,7 +22,7 @@ def __new__(cls):
 
     def __init__(self):
         self.engine = create_engine(
-            "sqlite:///v2ex.sqlite",
+            "sqlite:///v2ex2.sqlite",
             echo=False,
             json_serializer=lambda x: json.dumps(x, ensure_ascii=False),
         )
@@ -53,6 +50,6 @@ def exist(
 
     def get_max_topic_id(self) -> int:
         result = self.session.execute(text("SELECT max(id) FROM topic")).fetchone()
-        if result is None:
+        if result is None or result[0] is None:
             return 1
         return int(result[0])