Python爬虫 DEMO 抓取JD 搜索页

最新推荐文章于 2025-07-29 19:45:28 发布

Teamoping

最新推荐文章于 2025-07-29 19:45:28 发布

阅读量277

点赞数

CC 4.0 BY-SA版权

文章标签： python

本文链接：https://blog.youkuaiyun.com/Teamoping/article/details/108234985

这是一个Python爬虫DEMO，用于抓取京东搜索页（以macbook pro为例）的商品信息，包括商品ID、名称和价格。程序使用requests库获取网页内容，BeautifulSoup解析HTML，最后将数据写入CSV文件。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

#!/usr/bin/env python
# -*- coding:utf-8 -*-

import requests
import csv
from requests.exceptions import RequestException
from bs4 import BeautifulSoup

def download(url, headers, num_retries=3):
print("download", url)
try:
response = requests.get(url, headers=headers)
print(response.status_code)
if response.status_code == 200:
return response.content

return None
except RequestException as e:
print(e.response)
html = ""
if hasattr(e.response, 'status_code'):
code = e.response.status_code
print('error code', code)
if num_retries > 0 and 500 <= code < 600:
html = download(url, headers, num_retries - 1)
else:
code = None
return html

def find_Computer(url, headers):
r = download(url, headers=headers)

page = BeautifulSoup(r, "lxml")

all_items = page.find_all('li', attrs={'class':'gl-item'})

with open("Computer.csv", 'w', newline='') as f:
writer = csv.writer(f)
fields = ('ID', '名称', '价格')
writer.writerow(fields)

for all in all_items:
Computer_id = all["data-sku"]
print(f"电脑ID为：{Computer_id}")

Computer_name = all.find('div', attrs={'class':'p-name p-name-type-2'}).find('em').text
print(f"电脑的名称为：{Computer_name}")

Computer_price = all.find('div', attrs={'class':'p-price'}).find('i').text
print(f"电脑的价格为：{Computer_price}元")

row = []
row.append(Computer_id)
row.append(Computer_name)
row.append(str(Computer_price) + "元")
writer.writerow(row)

def main():
headers = {
'User-agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
"referer": "https://passport.jd.com"
}
URL = "https://search.jd.com/Search?keyword=macbook%20pro&enc=utf-8&suggest=5.def.0.V09&wq=mac&pvid=d3b040ed68154b06a9ff9532a10b4cac"

find_Computer(URL, headers=headers)

if __name__ == '__main__':
main()