Python 爬取豆瓣Top250

Python 爬取豆瓣Top250

跟着哔哩哔哩里的 Python 257万播放量的老师学习，老师讲的很详细也很透彻，把代码贴出来为了提供一起学习的小伙伴们参考。

# -*- coding: utf-8 -*-
# @Author : llvyr
# @qq:994814645    import urllib.parse
import urllib.request
import urllib.error
from bs4 import BeautifulSoup
import urllib.parse
import re
import xlwtbaseturl = "https://movie.douban.com/top250?start="
savaPath = "豆瓣电影Top250.xls"# 电影的详情匹配规则
# 电影链接
findLink = re.compile(r'')
# 电影图片
findImg = re.compile(r'(.*)')
# 电影评分
findRating = re.compile(r'(.*)')
# 电影评价人数
findCommentNo = re.compile(r'(\d*)人评价')
# 电影概括
findDescriptors = re.compile(r'(.*)')
# 电音的相关内容
findContent = re.compile(r'
(.*?)
', re.S)# 爬取单个网页def askUrl(baseturls):hesders = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36","Cookie": '换成自己的cookie值'}request = urllib.request.Request(url=baseturls, headers=hesders)html = ""try:response = urllib.request.urlopen(request)html = response.read().decode("utf-8")# print(html)except urllib.error.URLError as e:if hasattr(e, "code"):print(e.code)if hasattr(e, "reason"):print(e.reason)return html# 1. 爬取网页
def getData(baseturls):datalist = []for i in range(0, 10):url = baseturls + str(i * 25)html = askUrl(url)# 2. 解析数据suop = BeautifulSoup(html, "html.parser")for item in suop.find_all('div', class_="item"):data = []item = str(item)print(item)title = re.findall(findTitle, item)  # 名称if len(title) == 2:ctitle = title[0]  # 中文名data.append(ctitle)otitle = title[1].replace('/', " ")  # 外国名 替换 “/”# otitle = re.sub("\xa0", "", otitle)data.append(otitle)else:data.append(title[0])data.append(' ')link = re.findall(findLink, item)[0]  # 链接data.append(link)imgSrc = re.findall(findImg, item)[0]  # 封面data.append(imgSrc)rating = re.findall(findRating, item)[0]  # 评分data.append(rating)commentNo = re.findall(findCommentNo, item)[0]  # 评价人数data.append(commentNo)descriptors = re.findall(findDescriptors, item)  # 描述if len(descriptors) != 0:descriptors = descriptors[0].replace("。", "")data.append(descriptors)else:data.append(" ")content = re.findall(findContent, item)[0]  # 内容content = re.sub('(\s+)?', " ", content)  # 去掉 
content = re.sub("/", "", content)# content = re.sub("\xa0", "", content)data.append(content.strip())  # 去除前后的空格datalist.append(data)  # 处理好的一部电影信息放进数组中print(datalist)return datalist# 3. 存储数据
def savaData(datalist, savaPaths):workbook = xlwt.Workbook(encoding='utf-8', style_compression=0)worksheet = workbook.add_sheet("豆瓣电影Top250", cell_overwrite_ok=True)col = ("电影标题", "中文名称", "其他名称", "链接", "封面图片", "评分", "评价人数", "内容简介")for i in range(0, 8):worksheet.write(0, i, col[i])for j in range(0, 250):print(f"第{j + 1}条")data = datalist[j]for k in range(0, 8):worksheet.write(j + 1, k, data[k])workbook.save(savaPaths)  # 保存文档if __name__ == '__main__':getData(baseturl)datalists = getData(baseturl)# print(datalists)savaData(datalists, savaPath)

## 结果