多线程队列下载天涯帖子

版本一：

2. #coding:utf-8
4. import Queue
5. import threading
6. import time
7. import urllib2
8. import re
10. exitFlag = 0
14. class myThread (threading.Thread):
15. def __init__(self, threadID, name, q, txt_dict):
16. threading.Thread.__init__(self)
17. self.threadID = threadID
18. self.name = name
19. self.q = q
20. self.txt_dict = txt_dict
22. def run(self):
23. print "Starting " + self.name
24. process_data(self.name, self.q, self.txt_dict)
25. print "Exiting " + self.name
27. def process_data(threadName, q, txt_dict):
28. while not exitFlag:
29. queueLock.acquire()
30. if not workQueue.empty():
31. data = q.get()
32. queueLock.release()
33. print "%s processing %s" % (threadName, data)
34. down_text(txt_dict, data)
35. else:
36. queueLock.release()
37. time.sleep(1)
39. def down_text(txt_dict, url):
40. """根据传入的url抓出各页内容，按页数做键存入字典"""
41. html_content =urllib2.urlopen(url, timeout=30).read()
42. text_pattern = re.compile('<span>时间：(.*?)</span>.*?<!-- <div class="host-ico">楼主</div> -->.*?<div class="bbs-content.*?>\s*(.*?)</div>', re.DOTALL)
43. text = text_pattern.findall(html_content)
44. text_join = ['\r\n\r\n\r\n\r\n'.join(item) for item in text]
45. txt_dict[url] = text_join
48. def page(url):
49. """根据第一页地址抓取总页数"""
50. html_page = urllib2.urlopen(url, timeout=30).read()
51. page_pattern = re.compile(r'<a href="\S*?">(\d*)</a>\s*<a href="\S*?" class="\S*?">下页</a>')
52. page_result = page_pattern.search(html_page)
53. if page_result:
54. page_num = int(page_result.group(1))
55. return page_num
57. def write_text(filename, dict, urls):
58. """把字典内容按键（页数）写入文本，每个键值为每页内容的list列表"""
59. tx_file = open(filename, 'w+')
60. for i in urls:
61. print i
62. tx_list = dict[i]
63. for tx in tx_list:
64. tx = tx.replace('<br>', '\r\n').replace('<br />', '\r\n').replace(' ', '')
65. tx_file.write(tx.strip()+'\r\n'*4)
66. tx_file.close()
68. threadList = ["Thread-1", "Thread-2", "Thread-3"]
69. queueLock = threading.Lock()
70. workQueue = Queue.Queue(200)
71. threads = []
72. threadID = 1
73. url = 'http://bbs.tianya.cn/post-16-956795-1.shtml'
74. file_name ='abc.txt'
75. my_page = page(url)
76. my_dict = {}
77. urls = []
78. page_range = range(1, my_page+1)
79. for num in page_range:
80. myurl = '%s%s.shtml' % (url[:-7], num)
81. urls.append(myurl)
84. # 创建新线程
85. for tName in threadList:
86. thread = myThread(threadID, tName, workQueue, my_dict)
87. thread.start()
88. threads.append(thread)
89. threadID += 1
91. # 填充队列
92. queueLock.acquire()
93. for word in urls:
94. workQueue.put(word)
95. queueLock.release()
97. # 等待队列清空
98. while not workQueue.empty():
99. pass
101. # 通知线程是时候退出
102. exitFlag = 1
104. # 等待所有线程完成
105. for t in threads:
106. t.join()
107. print "Exiting Main Thread"
109. write_text(file_name, my_dict, urls)

版本二：

1. #coding:utf-8
3. import Queue
4. import threading
5. import time
6. import urllib2
7. import re
8. import sys
10. queue = Queue.Queue()
11. url = 'http://bbs.tianya.cn/post-16-996521-1.shtml'
12. urls=[]
13. Dict_txt = {}
14. file_name = 'abc.txt'
16. class ThreadNum(threading.Thread):
17. def __init__(self, queue):
18. threading.Thread.__init__(self)
19. self.queue = queue
21. def run(self):
22. while True:
23. #消费者端，从队列中获取num
24. num = self.queue.get()
25. sys.stdout.flush()
26. down_text(Dict_txt, num)
27. print '**finished download %s**' % num
28. time.sleep(1)
29. #在完成这项工作之后，使用 queue.task_done() 函数向任务已经完成的队列发送一个信号
30. self.queue.task_done()
33. def down_text(txt_dict, url):
34. """根据传入的url抓出各页内容，按页数做键存入字典"""
35. print '--downling %s--' % url
36. html_content =urllib2.urlopen(url, timeout=30).read()
37. text_pattern = re.compile('<span>时间：(.*?)</span>.*?<!-- <div class="host-ico">楼主</div> -->.*?<div class="bbs-content.*?>\s*(.*?)</div>', re.DOTALL)
38. text = text_pattern.findall(html_content)
39. text_join = ['\r\n\r\n\r\n\r\n'.join(item) for item in text]
40. txt_dict[url] = text_join
42. def page(url):
43. """根据第一页地址抓取总页数"""
44. html_page = urllib2.urlopen(url, timeout=30).read()
45. page_pattern = re.compile(r'<a href="\S*?">(\d*)</a>\s*<a href="\S*?" class="\S*?">下页</a>')
46. page_result = page_pattern.search(html_page)
47. list1 = []
48. if page_result:
49. page_num = int(page_result.group(1))
50. page_range = range(1, page_num+1)
51. for num in page_range:
52. myurl = '%s%s.shtml' % (url[:-7], num)
53. list1.append(myurl)
54. return list1
56. def write_text(filename, dict, urls):
57. """把字典内容按键（页数）写入文本，每个键值为每页内容的list列表"""
58. tx_file = open(filename, 'w+')
59. for i in urls:
60. tx_list = dict[i]
61. for tx in tx_list:
62. tx = tx.replace('<br>', '\r\n').replace('<br />', '\r\n').replace(' ', '')
63. tx_file.write(tx.strip()+'\r\n'*4)
64. tx_file.close()
67. start = time.time()
68. def main():
69. #产生一个 threads pool, 并把消息传递给thread函数进行处理，这里开启10个并发
70. urls = page(url)
72. for i in range(5):
73. t = ThreadNum(queue)
74. t.start()
76. #往队列中填错数据
77. for num in urls:
78. queue.put(num)
79. #wait on the queue until everything has been processed
80. queue.join()
83. print '---- start write ----'
84. write_text(file_name, Dict_txt, urls)
86. main()
87. print"Elapsed Time: %s" % (time.time() - start)

--版本三

1. #coding:utf-8
3. import urllib
4. import re
5. import threading
6. import os, time
8. class Down_Tianya(threading.Thread):
9. """多线程下载"""
10. def __init__(self, url, num, dt):
11. threading.Thread.__init__(self)
12. self.url = url
13. self.num = num
14. self.txt_dict = dt
16. def run(self):
17. print 'downling from %s' % self.url
18. self.down_text()
20. def down_text(self):
21. """根据传入的url抓出各页内容，按页数做键存入字典"""
22. html_content =urllib.urlopen(self.url).read()
23. text_pattern = re.compile('<span>时间：(.*?)</span>.*?<!-- <div class="host-ico">楼主</div> -->.*?<div class="bbs-content.*?>\s*(.*?)</div>', re.DOTALL)
24. text = text_pattern.findall(html_content)
25. text_join = ['\r\n\r\n\r\n\r\n'.join(item) for item in text]
26. self.txt_dict[self.num] = text_join
32. def page(url):
33. """根据第一页地址抓取总页数"""
34. html_page = urllib.urlopen(url).read()
35. page_pattern = re.compile(r'<a href="\S*?">(\d*)</a>\s*<a href="\S*?" class="\S*?">下页</a>')
36. page_result = page_pattern.search(html_page)
37. if page_result:
38. page_num = int(page_result.group(1))
39. return page_num
43. def write_text(dict, fn):
44. """把字典内容按键（页数）写入文本，每个键值为每页内容的list列表"""
45. tx_file = open(fn, 'w+')
46. pn = len(dict)
47. for i in range(1, pn+1):
48. tx_list = dict[i]
49. for tx in tx_list:
50. tx = tx.replace('<br>', '\r\n').replace('<br />', '\r\n').replace(' ', '')
51. tx_file.write(tx.strip()+'\r\n'*4)
52. tx_file.close()
55. def main():
56. url = 'http://bbs.tianya.cn/post-16-996521-1.shtml'
57. file_name ='abc.txt'
58. my_page = page(url)
59. my_dict = {}
61. print 'page num is : %s' % my_page
63. threads = []
65. """根据页数构造urls进行多线程下载"""
66. for num in range(1, my_page+1):
67. myurl = '%s%s.shtml' % (url[:-7], num)
68. downlist = Down_Tianya(myurl, num, my_dict)
69. downlist.start()
70. threads.append(downlist)
72. """检查下载完成后再进行写入"""
73. for t in threads:
74. t.join()
76. write_text(my_dict, file_name)
78. print 'All download finished. Save file at directory: %s' % os.getcwd()
80. if __name__ == '__main__':
81. main()

来自为知笔记(Wiz)

转载于:https://www.cnblogs.com/highroom/p/da69c3af3d0b2be8b4db5946e2de99f1.html