Weibo_Crawler/main.py

from get_weibo_content.get_one_page import get_one_page
from get_weibo_content.get_content import get_content
from get_weibo_content.get_mid import get_mid
from get_weibo_content.get_time import get_time
from save_data.save_data_to_excel import get_one_page_excel, save_to_excel
import datetime
import time
import random


def run_from_time_a_2_time_b(keyword_temp, date_begin_temp, date_end_temp, proxy_temp, cookie_temp, user_agent_temp):
    begin_num = 1
    page_count = begin_num
    all_data = []
    while True:
        print('[-]' + date_begin_temp + '-' + date_end_temp + '-page_' + str(page_count) + ':开始爬取...')
        html = get_one_page(keyword_temp, page_count, date_begin_temp, date_end_temp, proxy_temp, cookie_temp,
                            user_agent_temp)
        weibo_content_str, flag, if_contains_keyword = get_content(html, keyword)
        if not if_contains_keyword:
            break
        weibo_mid_str = get_mid(html)
        weibo_time_str = get_time(html, flag)
        length = len(weibo_content_str)
        print('[-]' + date_begin_temp + '-' + date_end_temp + '-page_' + str(page_count) + ':爬取到' + str(length) + '条信息')
        one_page_data = get_one_page_excel(weibo_content_str, weibo_mid_str, weibo_time_str, length)
        all_data += one_page_data
        time.sleep(random.randint(3, 6))
        page_count += 1
    print('[-]' + date_begin_temp + '-' + date_end_temp + '-page_' + str(page_count) + ':爬取完毕')
    save_to_excel(all_data, keyword_temp, date_begin_temp, date_end_temp)


if __name__ == '__main__':
    keyword = input('[-]请输入检索话题:')
    date_str = input('[-]请输入需要查询的当天日期(格式：2021-07-01):')
    date = datetime.datetime.strptime(date_str, "%Y-%m-%d")
    # cookie = input('[-]请输入cookie:')
    cookie = 'SUB=_2A25N_5x1DeRhGeBO4lsY9y_Pyz-IHXVvAyQ9rDV8PUJbkNAfLWH8kW1NRYEkPnoC7fl3RMKtu4E9iyGtx5ldeVcn; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWLS0lcQryz4UlBfKyjai.L5NHD95Qceh.41KMpe050Ws4Dqcjz-cyLdspDqgYt; SINAGLOBAL=2352245042816.5166.1627033753029; ULV=1627033753032:1:1:1:2352245042816.5166.1627033753029:; UOR=,,graph.qq.com; login_sid_t=5ed1466aa2327aac6d83a6652aa1a60a; cross_origin_proto=SSL; _s_tentry=passport.weibo.com; Apache=2352245042816.5166.1627033753029; WBtopGlobal_register_version=2021072412; webim_unReadCount=%7B%22time%22%3A1627119847912%2C%22dm_pub_total%22%3A1%2C%22chat_group_client%22%3A0%2C%22chat_group_notice%22%3A0%2C%22allcountNum%22%3A36%2C%22msgbox%22%3A0%7D; appkey=; SSOLoginState=1627122726; wvr=6; WBStorage=2ceabba76d81138d|undefined'
    user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0'
    proxy = '127.0.0.1:80'
    for i in range(0, 24):
        date_begin = date_str + '-' + str(i % 24)
        if i == 23:
            date_temp_str = datetime.datetime.strftime(date + datetime.timedelta(days=1), "%Y-%m-%d")
            date_end = date_temp_str + '-' + str((i + 1) % 24)
        else:
            date_end = date_str + '-' + str((i + 1) % 24)
        run_from_time_a_2_time_b(keyword, date_begin, date_end, proxy, cookie, user_agent)