python3 requests+bs4爬取某网页MM图片

最新推荐文章于 2022-05-09 00:02:27 发布

木马的翅膀

最新推荐文章于 2022-05-09 00:02:27 发布

阅读量1.1k

点赞数 1

分类专栏：学习笔记文章标签： python3 requests bs4 抓取图片

本文链接：https://blog.youkuaiyun.com/qq_33546486/article/details/85601502

版权

学习笔记专栏收录该内容

9 篇文章

订阅专栏

本文介绍了一个使用Python3的requests和bs4库实现的网页图片爬取案例，详细展示了如何从指定网页抓取MM图片，包括获取HTML页面、解析数据、创建文件夹以及保存图片的具体步骤。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

python3 requests+bs4爬取某网页MM图片

原理：
将所要抓取的首页分标题及地址保存到字典，遍历字典，对每一个标题下的所有分页进行抓取

import requests    
from bs4 import BeautifulSoup
import urllib
import os
import re
 
#获取网页Html页面
def getHTMLText(url):
    try:
        r=requests.get(url,timeout=30)
        r.raise_for_status()
        r.encoding=r.apparent_encoding
        return r.text
    except:
        return ""
 
#为图片创建文件夹并保存
def get_img(path,url):
    path = "D://e//" + path
    isExists=os.path.exists(path)
    if not isExists:
        os.makedirs(path) 
    i = 3
    while True:
        n = url.find("/",(len(url)-i))
        if n<0:
            i = i + 1
        else:
            break
    img = requests.get(url) 
    f = open(path+"//"+url[n+1:],'ab') 
    f.write(img.content)
    f.close()
    print("文件保存成功") 
def get_in_html(url) :
    start_url=url
    html=getHTMLText(start_url)
    soup = BeautifulSoup(html,"html.parser")
    pageInfo = soup.find(class_="itempage")
    ss  =str(pageInfo.contents[0])
    name = soup.h1.text
    page2 =int(ss[4:-7])
    tag_soup = soup.find(class_="picsbox picsboxcenter")
    imgUrl = tag_soup.img["src"]
    print("第1页")
    get_img(name,imgUrl)
    i = 2
    while i<=int(page2):
        temp_url = start_url[:-5] + "_" + str(i) + ".html"
        html=getHTMLText(temp_url)
        soup = BeautifulSoup(html,"html.parser")
        tag_soup = soup.find(class_="picsbox picsboxcenter")
        imgUrl = tag_soup.img["src"]
        print("第" + str(i) + "页")
        get_img(name,imgUrl)
        i = i+1

def main():
    
    pa = 1
    info = {}
    while pa<=20:
        start_url="https://******xiaohua/list_6_%d.html"%pa#7160
        html=getHTMLText(start_url)
        soup = BeautifulSoup(html,"html.parser")
        for k in soup.select("body > div > div.center > div > div.news_bom > div.news_bom-left > ul > ul > li > a"):
            info[(start_url[0:(start_url.find(".com")+4)]) + k['href']] = k['title']
        pa = pa + 1
    print(len (info))
    for k in info.keys():
        print("开始网页："+ k)
        get_in_html(k) 
 
main()