BeautifulSoup在爬虫中的使用实例

# 1.拿到主页面的源代码，然后提取到子页面的链接地址，href

# 2.通过href拿到子页面的内容，从子页面中找到图片的下载地址 img --> src

# 3.下载图片

import requests
from bs4 import BeautifulSoup
import time

url = "http://www.jj20.com/bz/nxxz/"
resp = requests.get(url)
resp.encoding = 'gbk'  # 处理乱码

# print(resp.text)
# 把源代码交给bs
main_page = BeautifulSoup(resp.text, "html.parser")
alist = main_page.find("ul", class_="picbz").find_all("a")
# print(alist)

for a in alist:
    href = a.get('href')  # 直接通过get就可以拿到属性的值
    # print(href)
    # 拿到子页面的源代码
    hreft = "http://www.jj20.com" + href
    # print(hreft)
    child_page_resp = requests.get(hreft)
    child_page_resp.encoding = 'gbk'
    child_page_text = child_page_resp.text
    # print(child_page_text)

    # 从子页面中拿到图片的下载路径
    child_page = BeautifulSoup(child_page_text, "html.parser")
    img = child_page.find("img")
    src = img.get("src")
    # print(src)

    # 下载图片
    img_resp = requests.get(src)
    # img_resp1= resp.content # 这里拿到的是字节
    img_name = src.split("/")[-1]
    # 拿到url中最后一个/以后的内容
    with open("img/" + img_name, mode="wb") as f:
        f.write(img_resp.content)  # 图片内容写入文件

    print("over!", img_name)