Python爬虫，批量下载小说网站上的小说！

爬虫脚本把这个小说网上的几乎所有小说都下载到了本地，一共27000+本小说，一共40G。

 

完整代码：

•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  
•  

#coding=utf-8 2 import urllib 3 import urllib2 4 import re 5 import os 6  7 webroot = 'http://www.xuanshu.com' 8  9 for page in range(20,220):10     print '正在下载第'+str(page)+'页小说'11     12     url = 'http://www.xuanshu.com/soft/sort02/index_'+str(page)+'.html'13     headers = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'  }14     try:15         request = urllib2.Request(url,headers=headers)16         response = urllib2.urlopen(request,timeout=180)17         #print response.read()18     except urllib2.URLError, e:19         if hasattr(e,"code"):20             print e.code21         if hasattr(e,"reason"):22             print e.reason23 24     html = response.read().decode('utf-8')25     #print html26     pattern = re.compile(u'<li>.*?<div class="s">.*?target="_blank">(.*?)</a><br />大小：(.*?)<br>.*?</em><br>更新：(.*?)</div>.*?<a href="(.*?)"><img.*?>(.*?)</a>.*?<div class="u">(.*?)</div>',re.S)27     items = re.findall(pattern,html)28     #print items29 30     for item in items:31         try:32             book_auther = item[0].encode('gbk')33             book_size = item[1].encode('gbk')34             book_updatetime = item[2].encode('gbk')35             book_link = item[3].encode('gbk')36             book_name = item[4].encode('gbk')37             book_note = item[5].encode('gbk')38             39             book_full_link = webroot + book_link    # 构建书的绝对地址40 41             #请求地址42             try:43                 request = urllib2.Request(book_full_link,headers=headers)44                 response = urllib2.urlopen(request,timeout=180)45             except urllib2.URLError, e:46                 if hasattr(e,"code"):47                     print e.code48                 if hasattr(e,"reason"):49                     print e.reason50             html = response.read().decode('utf-8')51             #print html52             pattern = re.compile('<a class="downButton.*?<a class="downButton" href=\'(.*?)\'.*?Txt.*?</a>',re.S)53             down_link = re.findall(pattern,html)54             print book_name55             print down_link56             57             # down txt58             try:59                 request = urllib2.Request(down_link[0].encode('utf-8'),headers=headers)60                 response = urllib2.urlopen(request,timeout=180)61             except urllib2.URLError, e:62                 if hasattr(e,"code"):63                     print e.code64                 if hasattr(e,"reason"):65                     print e.reason66             try:67                 fp = open(book_name+'.txt','w')68             except IOError,e:69                 pattern = re.compile('<strong>.*?>(.*?)<.*?</strong>',re.S)70                 book_name = re.findall(pattern,book_name)71                 fp = open(book_name[0]+'.txt','w')72             print 'start download'73             fp.write(response.read())74             print 'down finish\n'75             fp.close()76         except Exception,e:77             print '该条目解析出现错误，忽略'78             print e79             print ''80             fp = open('error.log','a')81             fp.write('page:'+str(page)+'\n')82             fp.write(item[4].encode('gbk'))83             #fp.write(e)84             fp.write('\nThere is an error in parsing process.\n\n')85             fp.close()

该脚本只定向抓取“选书网”小说站，“玄幻奇幻”分类下的小说。供网友们参考，可自行修改。

写得比较粗糙，勿喷……