SpiderMeet/run.py at main · AaronMrGuo/SpiderMeet · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
# from scrapy.cmdline import execute


# 定义要执行的爬虫名列表
spiders = ['beijingjiankangcujin',
           'henanyixuewang',
           'ningboyixuehui',
           'rentijiankangkejicujinhui',
           'sichuanshengyixuehui',
           'youmaihuixun',
           'zhongguokangaixiehui',
           'zhongguokangfuyixuehui',
           'zhongguoyaoxuehui',
           'zhonguoshengwuyixue',
           'xibaoshengwuxuexuehui']

def run_spiders():
    settings = get_project_settings()
    settings['LOG_ENABLED'] = False

    crawler_process = CrawlerProcess(settings)

    for spider in spiders:
        crawler_process.crawl(spider)

    crawler_process.start()
    print('-' * 200)

# 定义定时任务
# schedule.every().day.at("02:28").do(run_spiders)
#
# while True:
#     schedule.run_pending()
#     time.sleep(1)

if __name__ == '__main__':
    run_spiders()