正则爬取天堂图片网并存储在自动生成文件夹中

 

# -*- coding: utf-8 -*-
__author__ = '木之易'
__date__ = '2018/8/7 20:17'

import os
import re

from urllib import request


class WeddingImageSpider(object):

    def __init__(self, t_id):
        # 拼接完整地址
        self.url = 'http://www.ivsky.com'+t_id
        self.title = 'images'
        self.html = ''
        self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0'}
        self.create_directry()
        self.count = 0

    def create_directry(self):
        """
        创建存放图片的文件夹
        :return:
        """
        # 获取源代码
        self.get_html(self.url)
        # 2.获取帖子标题
        pattern = re.compile(r'<title>(.*?)</title>', re.S)
        # print(pattern)
        rs = re.search(pattern, self.html)
        # 3.提取帖子标题，并赋值给self.title
        if rs:
            self.title = rs.group(1)
        # 4.判断文件/目录是否存在
        if not os.path.exists(self.title):
            # 没有self.title这个文件夹，创建这个文件夹
            # mkdir() 创建文件夹
            os.mkdir(self.title)

    def get_total(self):
        """获取下一页"""
        self.get_html(self.url)
        pattern = re.compile(r'<div class="pagelist">(.*?)</div>', re.S)
        res = re.search(pattern, self.html)
        # 若找到
        if res:
            ul_html = res.group()
            # 找到含图片网址和关键字的标签
            # links = re.findall(re.compile(r"<a class='page-next' href='(.*?)</a>", re.S), ul_html)
            links = re.findall(re.compile(r"<a.*?href='(.*?)'", re.S), ul_html)
            print(links)
            print(111111111111111)
            if 'index' in links[-1]:
                # 拼接下一页完整地址
                # self.url = 'http://www.ivsky.com' + links[0][0:len(links)-6]
                self.url = 'http://www.ivsky.com' + links[-1]
                return True
            else:
                return False
        else:
            return False

    def get_html(self, url):
        # 构建请求对象
        req = request.Request(url=url, headers=self.headers)
        # 发送请求
        response = request.urlopen(req)
        # 读取相应数据
        self.html = response.read().decode('utf-8', 'ignore')

    def parser_html(self):
        # 下载地址 href = "http://img.ivsky.com/img/tupian/pic/201802/20/shatan_hunli_zhuangshi.jpg?download"
        # 图片地址 src = "http://img.ivsky.com/img/tupian/t/201802/20/shatan_hunli_zhuangshi.jpg"

        # 1.准备正则
        pattern = re.compile(r'<div class="il_img.*?<img src="(.*?)"', re.S)
        # 2.从源代码中提取数据
        result = re.findall(pattern, self.html)
        # 3.遍历下载每一张图片
        for link in result:
            self.count += 1
            print('正在下载第%s张图片' % self.count)
            # 4.拼接图片完整的存放路径
            # 自然风光图片 - 自然风景图片 (天堂图片网)/meilidehaitan-004.jpg
            path = self.title + '/' + '%s.jpg' % self.count
            # 5.下载图片
            request.urlretrieve(link, path)

    def run(self):
        count = 0
        while True:
            self.get_html(self.url)
            count += 1
            print('.........正在爬取第%s页数据，请稍后........' % count)
            self.parser_html()
            if not self.get_total():
                print('图片爬取完毕')
                break

if __name__ == '__main__':

    wedding = WeddingImageSpider('/tupian/meinv_t50/')
    wedding.run()