Skip to content

Commit 2a41b68

Browse files
NanmiCoderNanmiCoder
authored
Merge pull request #590 from 2513502304/main
关于 issue #589 的增强方法
2 parents cb2ae6c + af5a393 commit 2a41b68

File tree

4 files changed

+37
-13
lines changed

4 files changed

+37
-13
lines changed

‎m_bilibili.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
2+
# 1. 不得用于任何商业用途。
3+
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
4+
# 3. 不得进行大规模爬取或对平台造成运营干扰。
5+
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
6+
# 5. 不得用于任何非法或不当的用途。
7+
#
8+
# 详细许可条款请参阅项目根目录下的LICENSE文件。
9+
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
10+
11+
# -*- coding: utf-8 -*-

‎media_platform/bilibili/core.py‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -164,10 +164,8 @@ async def search(self):
164164
task_list = []
165165
try:
166166
task_list = [self.get_video_info_task(aid=video_item.get("aid"), bvid="", semaphore=semaphore) for video_item in video_list]
167-
except Exception as e :
168-
utils.logger.warning(
169-
f"[BilibiliCrawler.search] error in the task list. The video for this page will not be included. {e}"
170-
)
167+
except Exception as e:
168+
utils.logger.warning(f"[BilibiliCrawler.search] error in the task list. The video for this page will not be included. {e}")
171169
video_items = await asyncio.gather(*task_list)
172170
for video_item in video_items:
173171
if video_item:
@@ -177,16 +175,19 @@ async def search(self):
177175
await self.get_bilibili_video(video_item, semaphore)
178176
page += 1
179177
await self.batch_get_video_comments(video_id_list)
180-
# 按照 START_DAY 至 END_DAY 按照每一天进行筛选，这样能够突破 1000 条视频的限制，最大程度爬取该关键词下的所有视频
178+
# 按照 START_DAY 至 END_DAY 按照每一天进行筛选，这样能够突破 1000 条视频的限制，最大程度爬取该关键词下每一天的所有视频
181179
else:
182180
for day in pd.date_range(start=config.START_DAY, end=config.END_DAY, freq='D'):
183181
# 按照每一天进行爬取的时间戳参数
184182
pubtime_begin_s, pubtime_end_s = await self.get_pubtime_datetime(start=day.strftime('%Y-%m-%d'), end=day.strftime('%Y-%m-%d'))
185183
page = 1
184+
#!该段 while 语句在发生异常时（通常情况下为当天数据为空时）会自动跳转到下一天，以实现最大程度爬取该关键词下当天的所有视频
185+
#!除了仅保留现在原有的 try, except Exception 语句外，不要再添加其他的异常处理！！！否则将使该段代码失效，使其仅能爬取当天一天数据而无法跳转到下一天
186+
#!除非将该段代码的逻辑进行重构以实现相同的功能，否则不要进行修改！！！
186187
while (page - start_page + 1) * bili_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:
187-
# ! Catch any error if response return nothing, go to next day
188+
#! Catch any error if response return nothing, go to next day
188189
try:
189-
# ! Don't skip any page, to make sure gather all video in one day
190+
#! Don't skip any page, to make sure gather all video in one day
190191
# if page < start_page:
191192
# utils.logger.info(f"[BilibiliCrawler.search] Skip page: {page}")
192193
# page += 1
@@ -205,11 +206,7 @@ async def search(self):
205206
video_list: List[Dict] = videos_res.get("result")
206207

207208
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
208-
task_list = []
209-
try:
210-
task_list = [self.get_video_info_task(aid=video_item.get("aid"), bvid="", semaphore=semaphore) for video_item in video_list]
211-
finally:
212-
pass
209+
task_list = [self.get_video_info_task(aid=video_item.get("aid"), bvid="", semaphore=semaphore) for video_item in video_list]
213210
video_items = await asyncio.gather(*task_list)
214211
for video_item in video_items:
215212
if video_item:

‎schema/tables.sql‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,11 @@ CREATE TABLE `bilibili_video`
1616
`desc` longtext COMMENT '视频描述',
1717
`create_time` bigint NOT NULL COMMENT '视频发布时间戳',
1818
`liked_count` varchar(16) DEFAULT NULL COMMENT '视频点赞数',
19+
`disliked_count` varchar(16) DEFAULT NULL COMMENT '视频点踩数',
1920
`video_play_count` varchar(16) DEFAULT NULL COMMENT '视频播放数量',
21+
`video_favorite_count` varchar(16) DEFAULT NULL COMMENT '视频收藏数量',
22+
`video_share_count` varchar(16) DEFAULT NULL COMMENT '视频分享数量',
23+
`video_coin_count` varchar(16) DEFAULT NULL COMMENT '视频投币数量',
2024
`video_danmaku` varchar(16) DEFAULT NULL COMMENT '视频弹幕数量',
2125
`video_comment` varchar(16) DEFAULT NULL COMMENT '视频评论数量',
2226
`video_url` varchar(512) DEFAULT NULL COMMENT '视频详情URL',
@@ -35,6 +39,8 @@ CREATE TABLE `bilibili_video_comment`
3539
`id` int NOT NULL AUTO_INCREMENT COMMENT '自增ID',
3640
`user_id` varchar(64) DEFAULT NULL COMMENT '用户ID',
3741
`nickname` varchar(64) DEFAULT NULL COMMENT '用户昵称',
42+
`sex` varchar(64) DEFAULT NULL COMMENT '用户性别',
43+
`sign` varchar(64) DEFAULT NULL COMMENT '用户签名',
3844
`avatar` varchar(255) DEFAULT NULL COMMENT '用户头像地址',
3945
`add_ts` bigint NOT NULL COMMENT '记录添加时间戳',
4046
`last_modify_ts` bigint NOT NULL COMMENT '记录最后修改时间戳',
@@ -57,6 +63,8 @@ CREATE TABLE `bilibili_up_info`
5763
`id` int NOT NULL AUTO_INCREMENT COMMENT '自增ID',
5864
`user_id` varchar(64) DEFAULT NULL COMMENT '用户ID',
5965
`nickname` varchar(64) DEFAULT NULL COMMENT '用户昵称',
66+
`sex` varchar(64) DEFAULT NULL COMMENT '用户性别',
67+
`sign` varchar(64) DEFAULT NULL COMMENT '用户签名',
6068
`avatar` varchar(255) DEFAULT NULL COMMENT '用户头像地址',
6169
`add_ts` bigint NOT NULL COMMENT '记录添加时间戳',
6270
`last_modify_ts` bigint NOT NULL COMMENT '记录最后修改时间戳',
@@ -537,4 +545,4 @@ CREATE TABLE `zhihu_creator` (
537545
alter table douyin_aweme_comment add column `like_count` varchar(255) NOT NULL DEFAULT '0' COMMENT '点赞数';
538546

539547
alter table xhs_note add column xsec_token varchar(50) default null comment '签名算法';
540-
alter table douyin_aweme_comment add column `pictures` varchar(500) NOT NULL DEFAULT '' COMMENT '评论图片列表';
548+
alter table douyin_aweme_comment add column `pictures` varchar(500) NOT NULL DEFAULT '' COMMENT '评论图片列表';

‎store/bilibili/__init__.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,11 @@ async def update_bilibili_video(video_item: Dict):
5454
"nickname": video_user_info.get("name"),
5555
"avatar": video_user_info.get("face", ""),
5656
"liked_count": str(video_item_stat.get("like", "")),
57+
"disliked_count": str(video_item_stat.get("dislike", "")),
5758
"video_play_count": str(video_item_stat.get("view", "")),
59+
"video_favorite_count": str(video_item_stat.get("favorite", "")),
60+
"video_share_count": str(video_item_stat.get("share", "")),
61+
"video_coin_count": str(video_item_stat.get("coin", "")),
5862
"video_danmaku": str(video_item_stat.get("danmaku", "")),
5963
"video_comment": str(video_item_stat.get("reply", "")),
6064
"last_modify_ts": utils.get_current_timestamp(),
@@ -73,6 +77,8 @@ async def update_up_info(video_item: Dict):
7377
saver_up_info = {
7478
"user_id": str(video_item_card.get("mid")),
7579
"nickname": video_item_card.get("name"),
80+
"sex": video_item_card.get("sex"),
81+
"sign": video_item_card.get("sign"),
7682
"avatar": video_item_card.get("face"),
7783
"last_modify_ts": utils.get_current_timestamp(),
7884
"total_fans": video_item_card.get("fans"),
@@ -105,6 +111,8 @@ async def update_bilibili_video_comment(video_id: str, comment_item: Dict):
105111
"content": content.get("message"),
106112
"user_id": user_info.get("mid"),
107113
"nickname": user_info.get("uname"),
114+
"sex": user_info.get("sex"),
115+
"sign": user_info.get("sign"),
108116
"avatar": user_info.get("avatar"),
109117
"sub_comment_count": str(comment_item.get("rcount", 0)),
110118
"last_modify_ts": utils.get_current_timestamp(),

0 commit comments

Comments
 (0)