小项目 爬虫爬取图片 --站长素材网

要点: xpath
懒加载问题

t urllib.request
import urllib.parse
import time
from lxml import etree
import os

def handle_request(url,page):
# 第一页页码url单独判断
if page==1:
url = url.format(’’)
else:
url = url.format(’_’+str(page))
# print(url)
headers = {
‘User-Agent’ : ‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.3.4000’
}
request = urllib.request.Request(url=url,headers=headers)
return request
def download_img(image_src):
dirpath = ‘xinggan’
if not os.path.exists(dirpath):
os.mkdir(dirpath)
# 文件名
filename = os.path.basename(image_src)
# 搞图片路径
filepath = os.path.join(dirpath,filename)
# 发送请求保存图片
headers = {
‘User-Agent’ : ‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.3.4000’
}
request = urllib.request.Request(url=image_src,headers=headers)
response = urllib.request.urlopen(request)
with open(filepath,‘wb’) as fp:
fp.write(response.read())

def parse_content(content):
# 解析内容 获取图片
tree = etree.HTML(content)
image_list = tree.xpath(’//div[@id=“container”]/div/div/a/img/@src2’)
# print(image_list)
# print(len(image_list))
for image_src in image_list:
download_img(image_src)
def main():
url = ‘http://sc.chinaz.com/tupian/xingganmeinvtupian{}.html
# url = http://sc.chinaz.com/tupian/xingganmeinvtupian_3.html
start_page = int(input(‘请输入起始页码:’))
end_page = int(input(‘请输入结束页码:’))
for page in range(start_page,end_page+1):
request = handle_request(url,page)
content = urllib.request.urlopen(request).read().decode()
parse_content(content)
time.sleep(2)

if name == ‘main’:
main()

评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值