python爬虫_获取知乎最多100篇文章

这几天经常上知乎，觉得里面有些文章或者回答确实不错。就花了晚上时间写了这个爬虫。以前没有用Python写过独立的程序，所以这个程序bug比较多。现在贴出的代码可以运行，会在同级目录上生成zhihu_jingxuan.txt，该txt中就是爬取的文章。主要的问题是，当爬取的文章过多时，就会报超出最大循环嵌套数的错误。简单的查了一下，python最大允许的循环前套数是10000。用到了beautifulsoup库，觉得它里面获取标签的时候应该是用了迭代，导致超出了最大循环数。再次记录一下，有空看看源码。

[python] view plain copy

print ?

1. #coding:utf-8  
2. import urllib  
3. from bs4 import BeautifulSoup  
4. import re  
5.   
6. url = ”http://www.zhihu.com”  
7. filename = ”zhihu_jingxuan.txt”  
8.   
9. def parseArticleFromHtml(html):  
10.     soup = BeautifulSoup(html)  
11.     result = ”<<”+soup.html.head.title.string+“>>\r\n”  
12.       
13.     for i in soup.findAll(‘div’,{‘class’:‘zm-editable-content’}):  
14.         tmp = i  
15.         if tmp is not None:  
16.             tmp2 = str(tmp)  
17.             tmp3 = re.sub(’<[^>]+>’,“\r\n”,tmp2)  
18.             result += ”*************************\r\n”  
19. #           try:  
20.             result += tmp3  
21.             result +=”\r\n”           
22. #           except:  
23. #               continue  
24.     result +=”<><><><><><><><><><>”  
25.     for ii in range(5):  
26.         result = result.replace(”\r\n\r\n”,“\r\n”)  
27.     return result  
28.   
29. def parseArticleFromLink(link):  
30.     print link  
31.     html = urllib.urlopen(link)  
32.     content = html.read()  
33.     html.close()  
34.   
35. #   try:  
36.     article_string = parseArticleFromHtml(content)  
37.     myfilewriter = file(filename,’a+’)    
38.     myfilewriter.write(”\r\n”)  
39.     myfilewriter.write(article_string)    
40.     myfilewriter.close()  
41. #   except UnicodeEncodeError:  
42. #       pass  
43.       
44.     return  
45.   
46. mylist = []  
47. html = urllib.urlopen(url)  
48. content = html.read()  
49. html.close()  
50. soup = BeautifulSoup(content)  
51. info_cards = soup.findAll(’a’,{‘class’:‘rep’})  
52. for an_info_cards in info_cards:  
53.     print an_info_cards.span.string  
54.     newlink = url+dict(an_info_cards.attrs)[”href”]  
55.     newhtml = urllib.urlopen(newlink)  
56.     newcontent = newhtml.read()  
57.     newhtml.close()  
58.     newsoup = BeautifulSoup(newcontent)  
59.     question_links = newsoup.findAll(’a’,{‘class’:‘question_link’})  
60.     for a_question_link in question_links:  
61.         article_link = url+dict(a_question_link.attrs)[”href”]  
62. #         parseArticleFromLink(article_link)  
63.         if “answer” in article_link:  
64.             mylist.append(article_link)  
65.   
66.   
67. print len(mylist)  
68. counter = 100  
69. if(len(mylist)>counter):  
70.     for item in range(counter):  
71.         print item  
72.         parseArticleFromLink(mylist[item])  
73. else:  
74.     for item in mylist:  
75.         parseArticleFromLink(item)  

#coding:utf-8
import urllib
from bs4 import BeautifulSoup
import re

url = "http://www.zhihu.com"
filename = "zhihu_jingxuan.txt"

def parseArticleFromHtml(html):
    soup = BeautifulSoup(html)
    result = "<<"+soup.html.head.title.string+">>\r\n"

    for i in soup.findAll('div',{'class':'zm-editable-content'}):
        tmp = i
        if tmp is not None:
            tmp2 = str(tmp)
            tmp3 = re.sub('<[^>]+>',"\r\n",tmp2)
            result += "*************************\r\n"

            result += tmp3
            result +="\r\n"         

    result +="<><><><><><><><><><>"
    for ii in range(5):
        result = result.replace("\r\n\r\n","\r\n")
    return result

def parseArticleFromLink(link):
    print link
    html = urllib.urlopen(link)
    content = html.read()
    html.close()


    article_string = parseArticleFromHtml(content)
    myfilewriter = file(filename,'a+')  
    myfilewriter.write("\r\n")
    myfilewriter.write(article_string)  
    myfilewriter.close()


    return

mylist = []
html = urllib.urlopen(url)
content = html.read()
html.close()
soup = BeautifulSoup(content)
info_cards = soup.findAll('a',{'class':'rep'})
for an_info_cards in info_cards:
    print an_info_cards.span.string
    newlink = url+dict(an_info_cards.attrs)["href"]
    newhtml = urllib.urlopen(newlink)
    newcontent = newhtml.read()
    newhtml.close()
    newsoup = BeautifulSoup(newcontent)
    question_links = newsoup.findAll('a',{'class':'question_link'})
    for a_question_link in question_links:
        article_link = url+dict(a_question_link.attrs)["href"]

        if "answer" in article_link:
            mylist.append(article_link)


print len(mylist)
counter = 100
if(len(mylist)>counter):
    for item in range(counter):
        print item
        parseArticleFromLink(mylist[item])
else:
    for item in mylist:
        parseArticleFromLink(item)