2014-12-22

最新推荐文章于 2024-08-06 17:10:01 发布

原创最新推荐文章于 2024-08-06 17:10:01 发布 · 1.7k 阅读

0 ·

CC 4.0 BY-SA版权

本文介绍了一种使用Python从豆瓣小组抓取图片的方法。通过解析网页获取帖子链接，并进一步下载帖子内的图片。代码中涉及了网页请求、正则表达式匹配及多线程处理等内容。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1 http://www.gamefromscratch.com/

game from scratch

2 http://www.ccs.neu.edu/home/matthias/HtDP2e/part_prologue.html

racket lang

3 http://www.youkuaiyun.com/article/2014-12-17/2823183

sina post system

4 https://gitcafe.com/gdgeek/UnityGame

a unity3D game

5 http://pdos.csail.mit.edu/6.828/2011/schedule.html

a mit os course

import urllib
import re
import time
import threading


def getHtml(url):
	html = urllib.urlopen(url).read().decode('utf-8')
	return html


def gettopic(html):
	reg = r'http://www.douban.com/group/topic/\d+'
	topiclist = re.findall(reg,html)
	x = 0
	for topicurl in topiclist:
		x+=1
	return topicurl


def download(topic_page):
	reg2 = r'http://img3.douban.com/view/group_topic/large/public/.+\.jpg'
	imglist = re.findall(reg2,topic_page)
	i = 1
	print(imglist)
	download_img = None
	for imgurl in imglist:
		img_numlist = re.findall(r'p\d{7}',imgurl)
		print(img_numlist)
		for img_num in img_numlist:
			download_img = urllib.urlretrieve(imgurl,'/home/space/hua.song/mycode/girl/%s.jpg'%img_num)
			time.sleep(1)
			i+=1
			print(imgurl)


	return download_img


def main_loop(num,num_end,pagenum):
	while num <= num_end:
		html=getHtml('http://www.douban.com/group/kaopulove/discussion?start=%d'%num)
	#	print(html)
		topicurl = gettopic(html)
	#	print(topicurl)
		topic_page = getHtml(topicurl)
	#	print(topic_page)
		download_img = download(topic_page)
		num = page_num*25
		page_num+=1
	else:
		print('over')




page_end = int(input('input eng pagenum:'))
num_end = page_end*25
num = 0
page_num = 1
threadnum = 5


while num <= num_end:
	num_start = num
	num_stop = num + page_end / threadnum * 25
	pagenum = num / 25 + 1
	
	t = threading.Thread(target=main_loop,args=(num_start,num_stop,pagenum,))
	t.start()
	t.join()
	num = num_stop




	
#while num <= num_end:
#	html=getHtml('http://www.douban.com/group/kaopulove/discussion?start=%d'%num)
#	print(html)
#	topicurl = gettopic(html)
#	print(topicurl)
#	topic_page = getHtml(topicurl)
#	print(topic_page)
#	download_img = download(topic_page)
#	num = page_num*25
#	page_num+=1
#else:
#	print('over')