Python爬虫技术5天速成（2020全新合集），跟该老师学习写的爬虫代码（1）用xlwt

#@Author:king
#@File: spider.py
#-*- coding = utf-8 -*-
#@Time: 2020/5/18 16:52
#@Software: PyCharm

from bs4 import BeautifulSoup
import re
import urllib.request,urllib.error
import xlwt
import sqlite3

def main():
    baseurl="http://movie.douban.com/top250?start="
    #1 爬取网页
    datalist = getData(baseurl)
    #2 逐一解析数据 在getData函数中实现

    savepath = ".\\豆瓣电影top250.xls"
    #3 保存数据
    saveData(datalist,savepath)
    #askURL(baseurl)
#影片链接
findLink = re.compile(r'<a href="(.*?)">')
#影片图片
findImgSrc = re.compile(r'<img.*src="(.*?)"',re.S)#让换行符包含在匹配
#影片名字
findTitle = re.compile(r'<span class="title">(.*)</span>')
#影片评分
findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>')
#评价人数
findJudge = re.compile(r'<span>(\d*)人评价</span>')
#找电影简介
findInq = re.compile(r'<span class="inq">(.*)</span>')
#找到电影相关内容，像演员导演等
findBd = re.compile(r'<p class="">(.*?)</p>',re.S)
def getData(baseurl):#爬取网页
    datalist = []
    for i in range(0,10):#调用获取信息函数十次
        url = baseurl + str(i * 25)
        html = askURL(url) #保存获取到的网页源码

    # 2 逐一解析数据
        soup = BeautifulSoup(html,"html.parser")#用html的解析器解析
        for item in soup.find_all("div",class_="item"):  #查找符合一定条件的字符串形成列表
            #print(item)   测试查找所有电影信息
            data = [] # 保存所要电影信息
            item = str(item)


            link = re.findall(findLink,item)[0]#  影片链接获取
            data.append(link)
            imgSrc = re.findall(findImgSrc,item)[0]#  图片
            data.append(imgSrc)
            titles = re.findall(findTitle,item)
            if(len(titles)==2):
                ctitle = titles[0]
                data.append(ctitle)  #添加中文名
                otitle = titles[1].replace("/","") #去掉/
                data.append(otitle) #添加外文名

            else:
                data.append(titles[0])
                data.append(" ")  #留空，外国名留空

            rating = re.findall(findRating,item)[0]
            data.append(rating)
            judgeNum = re.findall(findJudge,item)[0]
            data.append(judgeNum)
            inq = re.findall(findInq,item)
            if len(inq)!=0:
                inq =inq[0].replace("。","")#去掉句号
                data.append(inq)
            else:
                data.append(" ")  # 留空

            bd = re.findall(findBd,item)[0]
            bd = re.sub('<br(\s+)?/>(\s+)',"",bd) #去掉<br/>
            bd = re.sub('/',"",bd) #替换/
            data.append(bd.strip()) #去除前后空格

            datalist.append(data) #存入每部电影信息
    print(datalist)
    return datalist


#得到指定url的网页内容
def askURL(url):
    #模拟浏览器头部信息，像豆瓣服务器发送消息
    head = {"User-Agent":" Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"}
        #用户代理表示告诉豆瓣服务器我们是什么类型的机器（本质上是该苏浏览器我们可以接收什么样水平的文件内容）
    request = urllib.request.Request(url=url,headers=head)
    html = "" #作为字符串保存
    try:
        response = urllib.request.urlopen(request)
        html = response.read().decode("utf-8")
        #print(html)

    except urllib.error.URLError as e:
        if hasattr(e,"code"):
            print(e.code)
        if hasattr(e,"reason"):
            print(e.reason)

    return html


def saveData(datalist,savepath):#保存数据
    print("save.......")
    workbook = xlwt.Workbook(encoding='utf-8',style_compression=0)
    worksheet = workbook.add_sheet("豆瓣电影top250",cell_overwrite_ok=True)
    col = ("电影链接","图片详情","电影中文名","电影外文名","评分","评分人数","简介","电影信息")
    for i in range(0,8):
        worksheet.write(0,i,col[i])#列名
    for i in range(0,250):
        print("第%d条信息"%(i+1))
        data = datalist[i]
        for j in range(0,8):
            worksheet.write(i+1,j,data[j])  #数据

    workbook.save(savepath)


if __name__ == '__main__':
    main()
    print("爬取完毕")