python爬取全书网小说

使用python爬取全书网小说

#-*- coding:UTF-8 -*- #编码设置
#全书网获取小说
from urllib2 import urlopen
import re

#小说网址
first_url = 'http://www.quanshuwang.com/book/9/9055'
#需要转码，否则乱码
html = urlopen(first_url).read().decode('gbk')
#print html

#保存小说名称和作者
novel_info = {}
novel_info['title'] = re.findall(r'<div class="chapName">.*?<strong>(.*?)</strong>',html)[0]
novel_info['author'] = re.findall(r'<div class="chapName"><span class="r">(.*?)</span><strong>',html)[0]

#print novel_info['title'],novel_info['author']
#获取小说章节主体div
novel_div = re.findall(r'<DIV class="clearfix dirconone">(.*?)</div>',html,re.S|re.I)[0]
#print novel_div
#获取小说每个章节对应的超链接
target_a = re.findall(r'<a.*?</a>',novel_div)
if target_a:
    for a in target_a:
        #print a
        href = re.findall(r'href="(.*?)"',a)[0]
        #print href
        #获取小说每个章节对应的内容
        content = urlopen(href).read().decode('gbk')
        print content
        #在这里同上使用正则获取小说正文，再把换行符、空格等字符去掉，保存即可