天涯论坛爬虫实战-优快云博客

本文提供三种不同版本的Python爬虫代码实现，用于抓取天涯论坛帖子的多页内容，并将其保存到本地文件中。涉及多线程技术、正则表达式及网页解析等关键技术。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

版本一：

#coding:utf-8
import Queue
import threading
import time
import urllib2
import re
exitFlag = 0
class myThread (threading.Thread):
def __init__(self, threadID, name, q, txt_dict):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.q = q
self.txt_dict = txt_dict
def run(self):
print "Starting " + self.name
process_data(self.name, self.q, self.txt_dict)
print "Exiting " + self.name
def process_data(threadName, q, txt_dict):
while not exitFlag:
queueLock.acquire()
if not workQueue.empty():
data = q.get()
queueLock.release()
print "%s processing %s" % (threadName, data)
down_text(txt_dict, data)
else:
queueLock.release()
time.sleep(1)
def down_text(txt_dict, url):
"""根据传入的url抓出各页内容，按页数做键存入字典"""
html_content =urllib2.urlopen(url, timeout=30).read()
text_pattern = re.compile('<span>时间：(.*?)</span>.*?.*?<div class="bbs-content.*?>\s*(.*?)</div>', re.DOTALL)
text = text_pattern.findall(html_content)
text_join = ['\r\n\r\n\r\n\r\n'.join(item) for item in text]
txt_dict[url] = text_join
def page(url):
"""根据第一页地址抓取总页数"""
html_page = urllib2.urlopen(url, timeout=30).read()
page_pattern = re.compile(r'<a href="\S*?">(\d*)</a>\s*<a href="\S*?" class="\S*?">下页</a>')
page_result = page_pattern.search(html_page)
if page_result:
page_num = int(page_result.group(1))
return page_num
def write_text(filename, dict, urls):
"""把字典内容按键（页数）写入文本，每个键值为每页内容的list列表"""
tx_file = open(filename, 'w+')
for i in urls:
print i
tx_list = dict[i]
for tx in tx_list:
tx = tx.replace('<br>', '\r\n').replace('<br />', '\r\n').replace(' ', '')
tx_file.write(tx.strip()+'\r\n'*4)
tx_file.close()
threadList = ["Thread-1", "Thread-2", "Thread-3"]
queueLock = threading.Lock()
workQueue = Queue.Queue(200)
threads = []
threadID = 1
url = 'http://bbs.tianya.cn/post-16-956795-1.shtml'
file_name ='abc.txt'
my_page = page(url)
my_dict = {}
urls = []
page_range = range(1, my_page+1)
for num in page_range:
myurl = '%s%s.shtml' % (url[:-7], num)
urls.append(myurl)
# 创建新线程
for tName in threadList:
thread = myThread(threadID, tName, workQueue, my_dict)
thread.start()
threads.append(thread)
threadID += 1
# 填充队列
queueLock.acquire()
for word in urls:
workQueue.put(word)
queueLock.release()
# 等待队列清空
while not workQueue.empty():
pass
# 通知线程是时候退出
exitFlag = 1
# 等待所有线程完成
for t in threads:
t.join()
print "Exiting Main Thread"
write_text(file_name, my_dict, urls)

版本二：

#coding:utf-8
import Queue
import threading
import time
import urllib2
import re
import sys
queue = Queue.Queue()
url = 'http://bbs.tianya.cn/post-16-996521-1.shtml'
urls=[]
Dict_txt = {}
file_name = 'abc.txt'
class ThreadNum(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while True:
#消费者端，从队列中获取num
num = self.queue.get()
sys.stdout.flush()
down_text(Dict_txt, num)
print '**finished download %s**' % num
time.sleep(1)
#在完成这项工作之后，使用 queue.task_done() 函数向任务已经完成的队列发送一个信号
self.queue.task_done()
def down_text(txt_dict, url):
"""根据传入的url抓出各页内容，按页数做键存入字典"""
print '--downling %s--' % url
html_content =urllib2.urlopen(url, timeout=30).read()
text_pattern = re.compile('<span>时间：(.*?)</span>.*?.*?<div class="bbs-content.*?>\s*(.*?)</div>', re.DOTALL)
text = text_pattern.findall(html_content)
text_join = ['\r\n\r\n\r\n\r\n'.join(item) for item in text]
txt_dict[url] = text_join
def page(url):
"""根据第一页地址抓取总页数"""
html_page = urllib2.urlopen(url, timeout=30).read()
page_pattern = re.compile(r'<a href="\S*?">(\d*)</a>\s*<a href="\S*?" class="\S*?">下页</a>')
page_result = page_pattern.search(html_page)
list1 = []
if page_result:
page_num = int(page_result.group(1))
page_range = range(1, page_num+1)
for num in page_range:
myurl = '%s%s.shtml' % (url[:-7], num)
list1.append(myurl)
return list1
def write_text(filename, dict, urls):
"""把字典内容按键（页数）写入文本，每个键值为每页内容的list列表"""
tx_file = open(filename, 'w+')
for i in urls:
tx_list = dict[i]
for tx in tx_list:
tx = tx.replace('<br>', '\r\n').replace('<br />', '\r\n').replace(' ', '')
tx_file.write(tx.strip()+'\r\n'*4)
tx_file.close()
start = time.time()
def main():
#产生一个 threads pool, 并把消息传递给thread函数进行处理，这里开启10个并发
urls = page(url)
for i in range(5):
t = ThreadNum(queue)
t.start()
#往队列中填错数据
for num in urls:
queue.put(num)
#wait on the queue until everything has been processed
queue.join()
print '---- start write ----'
write_text(file_name, Dict_txt, urls)
main()
print"Elapsed Time: %s" % (time.time() - start)

--版本三

#coding:utf-8
import urllib
import re
import threading
import os, time
class Down_Tianya(threading.Thread):
"""多线程下载"""
def __init__(self, url, num, dt):
threading.Thread.__init__(self)
self.url = url
self.num = num
self.txt_dict = dt
def run(self):
print 'downling from %s' % self.url
self.down_text()
def down_text(self):
"""根据传入的url抓出各页内容，按页数做键存入字典"""
html_content =urllib.urlopen(self.url).read()
text_pattern = re.compile('<span>时间：(.*?)</span>.*?.*?<div class="bbs-content.*?>\s*(.*?)</div>', re.DOTALL)
text = text_pattern.findall(html_content)
text_join = ['\r\n\r\n\r\n\r\n'.join(item) for item in text]
self.txt_dict[self.num] = text_join
def page(url):
"""根据第一页地址抓取总页数"""
html_page = urllib.urlopen(url).read()
page_pattern = re.compile(r'<a href="\S*?">(\d*)</a>\s*<a href="\S*?" class="\S*?">下页</a>')
page_result = page_pattern.search(html_page)
if page_result:
page_num = int(page_result.group(1))
return page_num
def write_text(dict, fn):
"""把字典内容按键（页数）写入文本，每个键值为每页内容的list列表"""
tx_file = open(fn, 'w+')
pn = len(dict)
for i in range(1, pn+1):
tx_list = dict[i]
for tx in tx_list:
tx = tx.replace('<br>', '\r\n').replace('<br />', '\r\n').replace(' ', '')
tx_file.write(tx.strip()+'\r\n'*4)
tx_file.close()
def main():
url = 'http://bbs.tianya.cn/post-16-996521-1.shtml'
file_name ='abc.txt'
my_page = page(url)
my_dict = {}
print 'page num is : %s' % my_page
threads = []
"""根据页数构造urls进行多线程下载"""
for num in range(1, my_page+1):
myurl = '%s%s.shtml' % (url[:-7], num)
downlist = Down_Tianya(myurl, num, my_dict)
downlist.start()
threads.append(downlist)
"""检查下载完成后再进行写入"""
for t in threads:
t.join()
write_text(my_dict, file_name)
print 'All download finished. Save file at directory: %s' % os.getcwd()
if __name__ == '__main__':
main()