python爬取邮件内容_python爬取邮箱

1 importurllib22 importre3 from pyquery importPyQuery as pq4 from lxml importetree5 importurlparse6 importtime7

8 allUrls =set()9 allMails =set()10 urlsDownlist =[]11

12 classmailCrawler:13 def __init__(self,mailExpression,start_url,maxcount):14 '''mailExpressoin 邮箱的正则表达式;15 start_url开始邮箱；16 maxcount最大数量'''

17 self.mailpattern =re.compile(mailExpression)18 self.maxcount =maxcount19 self.htmlcount =020 self.UrlsQlist = []#url queue 实现广度优先

21 self.url =start_url22

23

24 defurl_normal(self,url):25 '''url 规范化'''

26 scheme,netloc,path,query = urlparse.urlsplit(url)[:4]27 netloc =netloc.lower()28

29 url.encode("utf-8")30

31 ifpath:32 path = re.sub('/{2,}','/',path)#去除url中的重复/

33 path = re.sub(r'\.$','',path)#去除url中结尾多余的点

34 path = re.sub('/$','&#