2022年 11月 6日

Python爬取微博数据

Python爬取微博数据

一、需求说明

       抓取某一用户的新浪微博数据，抓取的内容包括：id号、微博发布的时间、正文（仅提取文字）、转发数、评论数、点赞数。

在返回的json数据结构中，我们需要的是以下字段：

  1. data
  2.   cards
  3.     mblog
  4.       id #id号
  5.       created_at # 发布时间
  6.       text # 正文
  7.       reposts_count # 转发数
  8.       comments_count # 评论数
  9.       attitudes_count # 点赞数

二、代码实现

        确保安装了requests和pyquery库。

  1. pip install pyquery
  2. pip install requests

       1）爬取微博数据写入weibo.txt文本中，具体代码解析：

  1. import requests
  2. from urllib.parse import urlencode
  3. from pyquery import PyQuery as pq
  4.  
  5. host = 'm.weibo.cn'
  6. base_url = 'https://%s/api/container/getIndex?' % host
  7. user_agent = 'User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 wechatdevtools/0.7.0 MicroMessenger/6.3.9 Language/zh_CN webview/0'
  8.  
  9. headers = {
  10.     'Host': host,
  11.     'Referer': 'https://m.weibo.cn/u/XXX',
  12.     'User-Agent': user_agent
  13. }
  14.  
  15.  
  16. # 按页数抓取数据
  17. def get_single_page(page):
  18.     params = {
  19.         'type': 'uid',
  20.         'value': XXX,
  21.         'containerid': YYY,
  22.         'page': page
  23.     }
  24.     url = base_url + urlencode(params)
  25.     try:
  26.         response = requests.get(url, headers=headers)
  27.         if response.status_code == 200:
  28.             return response.json()
  29.     except requests.ConnectionError as e:
  30.         print('抓取错误', e.args)
  31.  
  32.  
  33. # 解析页面返回的json数据
  34. def parse_page(json):
  35.     items = json.get('data').get('cards')
  36.     for item in items:
  37.         item = item.get('mblog')
  38.         if item:
  39.             data = {
  40.                 'id': item.get('id'),
  41.                 'created_at':item.get('created_at'),
  42.                 'text': pq(item.get("text")).text(),  # 仅提取内容中的文本
  43.                 'attitudes': item.get('attitudes_count'),
  44.                 'comments': item.get('comments_count'),
  45.                 'reposts': item.get('reposts_count')
  46.             }
  47.             yield data
  48.  
  49.  
  50. if __name__ == '__main__':
  51.     fw = open('./weibo.txt','w',encoding='UTF-8') #写入txt文件中
  52.     for page in range(1, 10):  # 抓取前十页的数据
  53.         json_r = get_single_page(page)
  54.         results = parse_page(json_r)
  55.         for result in results:
  56.             print(result)
  57.             fw.writelines(json.dumps((result),ensure_ascii=False))
  58.             fw.writelines('\n')
  59.     fw.close()

   其中value与containerid数据为进入所需要爬取微博号页面，右键->查看网页源代码中所对应数据。

  

  value = oid对应值，containerid = page_id对应值。

运行结果：

        2）读取weibo.txt文本中数据写入weibo.xls中，具体代码解析：

  1. def getExcel():
  2.     # 创建excel工作表
  3.     workbook = xlwt.Workbook(encoding='utf-8')
  4.     worksheet = workbook.add_sheet('Weibo')
  5.  
  6.     # 设置表头
  7.     worksheet.write(0, 0, label='id')
  8.     worksheet.write(0, 1, label='created_at')
  9.     worksheet.write(0, 2, label='text')
  10.     worksheet.write(0, 3, label='attitudes')
  11.     worksheet.write(0, 4, label='comments')
  12.     worksheet.write(0, 5, label='reposts')
  13.  
  14.     # 读取json文件
  15.     data = []
  16.     with open('./weibo.txt', 'r', encoding='UTF-8') as f:
  17.         for i in f.readlines():
  18.             data.append(json.loads(i))
  19.     # 将json字典写入excel
  20.     # 变量用来循环时控制写入单元格，感觉有更好的表达方式
  21.     val = 1
  22.     for list_item in data:
  23.         for key, value in list_item.items():
  24.             if key == "id":
  25.                 worksheet.write(val, 0, value)
  26.             elif key == "created_at":
  27.                 worksheet.write(val, 1, value)
  28.             elif key == "text":
  29.                 worksheet.write(val, 2, value)
  30.             elif key == "attitudes":
  31.                 worksheet.write(val, 3, value)
  32.             elif key == "comments":
  33.                 worksheet.write(val, 4, value)
  34.             elif key == "reposts":
  35.                 worksheet.write(val, 5, value)
  36.         val += 1
  37.     # 保存
  38.     workbook.save('./weibo.xls')

        运行结果：

       

3）代码整合完整实现如下：

  1. # -*- coding: utf-8 -*-
  2. import xlwt
  3. import requests
  4. from urllib.parse import urlencode
  5. from pyquery import PyQuery as pq
  6. from openpyxl import Workbook,load_workbook
  7. import json
  8.  
  9. host = 'm.weibo.cn'
  10. base_url = 'https://%s/api/container/getIndex?' % host
  11. user_agent = 'User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 wechatdevtools/0.7.0 MicroMessenger/6.3.9 Language/zh_CN webview/0'
  12.  
  13. headers = {
  14.     'Host': host,
  15.     'Referer': 'https://m.weibo.cn/u/XXX',
  16.     'User-Agent': user_agent
  17. }
  18.  
  19. # 按页数抓取数据
  20. def get_single_page(page):
  21.     params = {
  22.         'type': 'uid',
  23.         'value': XXX,
  24.         'containerid': YYY,
  25.         'page': page
  26.     }
  27.     url = base_url + urlencode(params)
  28.     try:
  29.         response = requests.get(url, headers=headers)
  30.         if response.status_code == 200:
  31.             return response.json()
  32.     except requests.ConnectionError as e:
  33.         print('抓取错误', e.args)
  34.  
  35. # 解析页面返回的json数据
  36. def parse_page(json):
  37.     items = json.get('data').get('cards')
  38.     for item in items:
  39.         item = item.get('mblog')
  40.         if item:
  41.             data = {
  42.                 'id': item.get('id'),
  43.                 'created_at':item.get('created_at'),
  44.                 'text': pq(item.get("text")).text(),  # 仅提取内容中的文本
  45.                 'attitudes': item.get('attitudes_count'),
  46.                 'comments': item.get('comments_count'),
  47.                 'reposts': item.get('reposts_count')
  48.             }
  49.             yield data
  50.  
  51. def getTXT():
  52.     """写入txt文件中"""
  53.     fw = open('./weibo.txt','w',encoding='UTF-8')
  54.     for page in range(1, 10):  # 抓取前十页的数据
  55.         json_r = get_single_page(page)
  56.         results = parse_page(json_r)
  57.         for result in results:
  58.             print(result)
  59.             fw.writelines(json.dumps((result),ensure_ascii=False))
  60.             fw.writelines('\n')
  61.     fw.close()
  62.  
  63. def getExcel():
  64.     # 创建excel工作表
  65.     workbook = xlwt.Workbook(encoding='utf-8')
  66.     worksheet = workbook.add_sheet('罗小黑')
  67.  
  68.     # 设置表头
  69.     worksheet.write(0, 0, label='id')
  70.     worksheet.write(0, 1, label='created_at')
  71.     worksheet.write(0, 2, label='text')
  72.     worksheet.write(0, 3, label='attitudes')
  73.     worksheet.write(0, 4, label='comments')
  74.     worksheet.write(0, 5, label='reposts')
  75.  
  76.     # 读取json文件
  77.     data = []
  78.     with open('./weibo.txt', 'r', encoding='UTF-8') as f:
  79.         for i in f.readlines():
  80.             data.append(json.loads(i))
  81.     # 将json字典写入excel
  82.     # 变量用来循环时控制写入单元格，感觉有更好的表达方式
  83.     val = 1
  84.     for list_item in data:
  85.         for key, value in list_item.items():
  86.             if key == "id":
  87.                 worksheet.write(val, 0, value)
  88.             elif key == "created_at":
  89.                 worksheet.write(val, 1, value)
  90.             elif key == "text":
  91.                 worksheet.write(val, 2, value)
  92.             elif key == "attitudes":
  93.                 worksheet.write(val, 3, value)
  94.             elif key == "comments":
  95.                 worksheet.write(val, 4, value)
  96.             elif key == "reposts":
  97.                 worksheet.write(val, 5, value)
  98.         val += 1
  99.     # 保存
  100.     workbook.save('./weibo.xls')
  101.  
  102. if __name__ == '__main__':
  103.     getTXT()
  104.     getExcel()