使用xpath爬取百度贴吧图片

最新推荐文章于 2020-12-05 02:56:53 发布

转载最新推荐文章于 2020-12-05 02:56:53 发布 · 345 阅读

本文介绍了一款使用Python编写的贴吧图片爬虫，利用requests、lxml和threading等库，实现多线程下载贴吧内的图片。代码适用于初学者，通过实例展示了如何解析网页源码，提取图片链接并进行高效下载。

此代码写的蛮不错的，研究了两天，适合初学者分析

import time
from lxml import etree
import requests
import os
import threading
 
def down_image(url,kw):
    data = requests.get(url).content
    name = url[-10:]
    print(name)
    print("正在下载%s...%s"%(name,url))
    print(url)
    if not os.path.exists("./" + kw):
        os.mkdir("./" + kw)
    with open("./%s/%s"%(kw,name),"wb") as f:
        f.write(data)
 
def load_image(link_list,kw):
    headers = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko)"}
    thread_list = []
    begin = time.time()
    print("开始下载图片...")
    for link in link_list:
        link = "https://tieba.baidu.com" + link
        html = requests.get(link,headers = headers).text
        xml = etree.HTML(html)
        link_list = xml.xpath("//div/img[@class = 'BDE_Image']/@src")
        for link in link_list:
            t = threading.Thread(target = down_image,args = (link,kw))
            t.start()
            thread_list.append(t)
        for t in thread_list:
            t.join()
    end = time.time()
    print("下载完成共耗时%f秒"%(end-begin))
 
def load_page(url,payload):
    headers = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko)"}
    html = requests.get(url,params = payload,headers = headers).text
    xml = etree.HTML(html)
    link_list = xml.xpath("//div[@class = 't_con cleafix']//div[@class = 'threadlist_lz clearfix']/div/a/@href")
    for link in link_list:
        link = "https://tieba.baidu.com" + link
    load_image(link_list,payload['kw']) 
 
def main():
    kw = input("请输入贴吧名称：")
    pagenum = int(input("请输入要爬取的页码："))
    payload = {'kw':kw,'pn':((pagenum-1)*50)}
    url = "https://tieba.baidu.com/f?ie=utf-8"
    load_page(url,payload)
 
thread_list = []
if __name__ == "__main__":
    main()


#原文：https://blog.youkuaiyun.com/sdzhr/article/details/80921115