通过搜索简单python爬虫爬取洋码头网站推荐页的商品和价格信息

本文链接：https://blog.youkuaiyun.com/weixin_42545347/article/details/81137549

本文介绍了一个使用Python实现的简单爬虫案例，该爬虫用于抓取洋码头网站上的商品信息，包括商品名称和价格等。文章提供了完整的源代码，并解释了如何配置请求头、处理代理服务器及异常情况。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

直接上源码：

# -*- coding: utf-8 -*-
"""
Created on Tue Jul 17 15:18:50 2018

@author: Administrator
"""
import random
import logging
import sys
from urllib import request
from urllib import error
import time
import re

#import jieba
# 获取logger的实例
logger = logging.getLogger("testLogger")

# 指定logger的输出格式
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')

# 文件日志，终端日志对象
file_handler = logging.FileHandler("testLogger.log")
# 文件日志按照指定的格式来写
file_handler.setFormatter(formatter)

console_handler = logging.StreamHandler(sys.stdout)
# 终端日志按照指定的格式来写
console_handler.setFormatter(formatter)

# 可以设置日志的级别
logger.setLevel(logging.INFO)
# 把文件日志，终端日志对象添加到日志处理器logger中
logger.addHandler(file_handler)
logger.addHandler(console_handler)

minRangeForProxy = 1
maxRangeForProxy = 1
headers = [('authority', 'www.ymatou.com'),
('method','GET'),
('path', '/products?k=%E9%A6%99%E5%A5%88%E5%84%BF'),
('scheme', 'https'),
('accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'),
#('accept-encoding','deflate, br'),
('accept-language', 'zh-CN,zh;q=0.9'),
('referer', 'http://www.ymatou.com/'),
('upgrade-insecure-requests', '1'),
('user-agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')]
yangmatou = 'http://www.ymatou.com/products?k='

def downloadHtml(url,headers=[],proxy={'https':''},
userProxyRate=6,timeout=None,decodeInfo="utf-8",num_retries=5):

'''
这是一个爬取网页的数据的函数
它支持设置HTTP Request Handers，能设置UA；
它支持代理服务器的设置
它支持timeout超时机制
它支持网页编码的指定
它支持服务器返回错误的处理
'''
if random.randint(minRangeForProxy,maxRangeForProxy) > userProxyRate:
proxy = None
print('no proxy')
proxy_handler = request.ProxyHandler(proxy)


opener = request.build_opener(proxy_handler)

opener.addheaders = headers
request.install_opener(opener)
html = None
try:
res = request.urlopen(url,timeout=timeout)
html = res.read().decode(decodeInfo)
except UnicodeDecodeError:
logger.error("UnicodeDecodeError")
except error.URLError or error.HTTPError as e:
logger.error('Download Error')
if num_retries >= 0:
time.sleep(random.randint(1,3))
if hasattr(e,'code') and 500 <=e.code<600:
html = downloadHtml(url,headers,proxy,userProxyRate,timeout,decodeInfo,num_retries-1)


finally:

return html

def getInfo(something,url= 'http://www.ymatou.com/products?k=',headers=
[('authority', 'www.ymatou.com'),
('method','GET'),
('path', '/products?k=%E9%A6%99%E5%A5%88%E5%84%BF'),
('scheme', 'https'),
('accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'),
#('accept-encoding','deflate, br'),
('accept-language', 'zh-CN,zh;q=0.9'),
('referer', 'http://www.ymatou.com/'),
('upgrade-insecure-requests', '1'),
('user-agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')]):
# something = '香奈儿'
string = request.quote(something)
url = url + string
headers[2] = ('path','/products?k='+string)
html = downloadHtml(url=url,headers=headers)
pattern = re.compile('class="product-img" title="([\s\S]*?)"[\s\S]*?¥(\d*)')
Info = re.findall(pattern,html)
return Info
#提取信息如下格式：
#<li class="product-item " module_index keyword="香奈儿" sproductid="a99b2c0f-1b6a-44a4-917f-8f1e8c8d2e0c">
 # <a href="//www.ymatou.com/product/a99b2c0f-1b6a-44a4-917f-8f1e8c8d2e0c.html" target="_blank" class="product-img" title="Chanel香奈儿丝绒唇釉雾面短管唇釉6ml 140色">
 # <img class="lazy" alt="Chanel香奈儿丝绒唇釉雾面短管唇釉6ml 140色" src="https://img-blog.csdnimg.cn/2022010613284265526.jpeg" data-src="http://pic1.ymatou.com/G02/shangou/M00/F2/B5/CgvUBVoWivWAIdQWAAGiFEV-j2Y000_1_1_n_w_l.jpg">
 # 
 #
 #
 # </a>
 # ¥219
 # 
 # 
 # 包邮包税
 # 
 # 
 # <a href="//www.ymatou.com/product/a99b2c0f-1b6a-44a4-917f-8f1e8c8d2e0c.html" target="_blank">Chanel香奈儿丝绒唇釉雾面短管唇釉6ml 140色</a>
 # <div class="seller-site">
 # <a class="seller sellerinfo" href="//www.ymatou.com/sellerhome/15206250">
 # <img src="http://pic1.ymatou.com/G02/M07/9F/DC/CgvUBFg0Bb-ALwKNAAAaZUTYnCc953_1_1_o.jpg" alt="加号全球购">
 # 
 # 
 # 
 # 加号全球购
 # </a>
 # <a class="site" href="javascript:void(0)">
 # <img src="http://img.ymatou.com/app/flag/circle/Japan.png" alt="Japan">
 # Japan
 # </a>
 # <div class="seller-info-wrap">
 # 
 # <div class="siw-hd">
 # 
 # 
 # 专业买手
 #
 # 
 # </div>
 # <div class="siw-bd">
 # 买家评分 
 # 4.8
 # 
 # <ul class="siw-r">
 # <li class="siw-item">客户服务
 # 4.9
 # </li>
 # <li class="siw-item">物流服务
 # 5.0
 # </li>
 # <li class="siw-item">综合评分
 # 4.9
 # 
 # 高于平均15.5%
 # 
 #
 # </li>
 # </ul>
 # </div>
# </div>
# </div>
# </li>

if __name__ == '__main__':
something = input('请输入要搜索的东西')
Info = getInfo(something)
with open('%s洋码头价格信息.txt'%something,'w') as f:
for i in Info:
print(i)
f.write('商品名称：%s：商品单价:%s'%(i[0],i[1])+'\n')

运行结果如图：