from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import TimeoutException, NoSuchElementException, WebDriverException
from bs4 import BeautifulSoup
import requests
import re
import time
import csv
import random
import os
import sys
from aip import AipNlp
from webdriver_manager.chrome import ChromeDriverManager
import pickle
from selenium.webdriver.common.action_chains import ActionChains
# 百度AI配置
APP_ID = '119348823'
API_KEY = 'BMUyFD1qn0p4BgaRL5ZsFAHS'
SECRET_KEY = 'jSvSCxAOm47OGB7JxM0g05UKWeagFFPe'
client = AipNlp(APP_ID, API_KEY, SECRET_KEY)
# 随机请求头
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.142 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15"
]
def get_driver():
"""创建并配置浏览器驱动"""
chrome_options = Options()
# 调试时注释掉无头模式
# chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--window-size=1920,1080")
chrome_options.add_argument(f"user-agent={random.choice(USER_AGENTS)}")
chrome_options.add_argument("--disable-blink-features=AutomationControlled")
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)
# 自动下载并管理ChromeDriver
service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=service, options=chrome_options)
# 隐藏自动化特征
driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
return driver
def login_jd(driver):
"""执行京东登录流程"""
print("开始登录京东账号...")
login_url = "https://passport.jd.com/new/login.aspx"
driver.get(login_url)
time.sleep(3)
# 检查是否已有cookies文件
cookies_file = "jd_cookies.pkl"
if os.path.exists(cookies_file):
print("检测到已有cookies文件，尝试加载...")
with open(cookies_file, 'rb') as f:
cookies = pickle.load(f)
for cookie in cookies:
driver.add_cookie(cookie)
print("cookies加载完成")
driver.refresh()
time.sleep(2)
# 检查登录是否成功
try:
nickname = driver.find_element(By.CLASS_NAME, "nickname").text
print(f"登录成功! 用户名: {nickname}")
return True
except:
print("cookies已失效，需要重新登录")
print("请手动完成登录...")
print("等待用户登录（你有60秒时间完成登录）")
# 等待登录完成
try:
WebDriverWait(driver, 60).until(
EC.presence_of_element_located((By.CLASS_NAME, "nickname"))
)
nickname = driver.find_element(By.CLASS_NAME, "nickname").text
print(f"登录成功! 用户名: {nickname}")
# 保存cookies
cookies = driver.get_cookies()
with open(cookies_file, 'wb') as f:
pickle.dump(cookies, f)
print("登录状态已保存到cookies文件")
return True
except TimeoutException:
print("登录超时，继续未登录状态爬取")
return False
def jd_book_crawler(search_keyword, max_pages=2):
"""修改后的京东图书爬虫函数 - 支持登录"""
print("启动浏览器...")
driver = get_driver()
# 执行登录
login_success = login_jd(driver)
books_data = []
try:
print(f"访问京东首页...")
driver.get("https://www.jd.com")
time.sleep(random.uniform(1, 3))
print("京东首页加载成功")
# 搜索图书
print(f"搜索关键词: {search_keyword}")
try:
# 多种定位搜索框的方式
search_box = WebDriverWait(driver, 15).until(
EC.element_to_be_clickable((By.ID, "key"))
)
except TimeoutException:
# 备选定位方式
search_box = WebDriverWait(driver, 15).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "input.search-key"))
)
# 模拟真实输入
search_box.clear()
for char in search_keyword:
search_box.send_keys(char)
time.sleep(random.uniform(0.05, 0.15))
# 提交搜索
search_box.send_keys(Keys.ENTER)
print("已提交搜索")
time.sleep(random.uniform(2, 4))
# 验证是否跳转到搜索结果页
try:
WebDriverWait(driver, 15).until(
EC.url_contains("search")
)
print("已进入搜索结果页")
except TimeoutException:
print("可能遇到验证码或反爬页面，尝试重新加载...")
driver.get(f"https://search.jd.com/Search?keyword={search_keyword}")
time.sleep(random.uniform(3, 5))
# 等待结果加载
print("等待商品列表加载...")
try:
# 多种等待商品加载的方式
WebDriverWait(driver, 20).until(
EC.presence_of_element_located((By.CSS_SELECTOR, ".gl-item, .goods-list-v2 .item, .j-sku-item"))
)
time.sleep(random.uniform(1, 2))
except TimeoutException:
print("商品加载超时，尝试备用加载方式...")
# 尝试滚动页面触发加载
driver.execute_script("window.scrollTo(0, document.body.scrollHeight/3);")
time.sleep(2)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight/2);")
time.sleep(2)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(3)
for page in range(1, max_pages + 1):
print(f"正在处理第 {page} 页")
# 获取页面源码
html = driver.page_source
soup = BeautifulSoup(html, 'html.parser')
# 多种商品列表选择器
items = soup.select('.gl-item') # 京东主站
if not items:
items = soup.select('.goods-list-v2 .item') # 图书频道
if not items:
items = soup.select('.j-sku-item') # 备用选择器
if not items:
print("警告：未找到商品元素，尝试保存页面源码以供分析")
with open(f"jd_page_{page}.html", "w", encoding="utf-8") as f:
f.write(html)
print("页面源码已保存")
continue
print(f"本页找到 {len(items)} 个商品")
for idx, item in enumerate(items):
try:
# 多种标题选择器
title_elem = item.select_one('.p-name a em') or \
item.select_one('.p-name a') or \
item.select_one('.p-name-type-2 a') or \
item.select_one('.name a')
title = title_elem.text.strip() if title_elem else "N/A"
# 多种价格选择器
price_elem = item.select_one('.p-price strong') or \
item.select_one('.p-price i') or \
item.select_one('.price-box .price') or \
item.select_one('.j-price')
price = price_elem.text.strip() if price_elem else "0.00"
# 提取SKU/ISBN
isbn = item.get('data-sku') or item.get('data-spu') or "N/A"
# 详情页URL
detail_elem = item.select_one('.p-img a') or \
item.select_one('.pic a') or \
item.select_one('.name a')
detail_url = detail_elem['href'] if detail_elem and 'href' in detail_elem.attrs else ""
if detail_url and not detail_url.startswith('http'):
detail_url = 'https:' + detail_url
books_data.append({
'title': title,
'price': price,
'isbn': isbn,
'url': detail_url
})
if (idx + 1) % 5 == 0:
print(f"已处理 {idx + 1}/{len(items)} 个商品")
except Exception as e:
print(f"商品 {idx + 1} 提取错误: {str(e)[:50]}...")
# 翻页处理
if page < max_pages:
print("尝试翻页...")
try:
# 多种翻页按钮定位方式
next_btn = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, '.pn-next, .pn-next:not(.disabled)'))
)
driver.execute_script("arguments[0].scrollIntoView({behavior: 'smooth', block: 'center'});",
next_btn)
time.sleep(0.5)
driver.execute_script("arguments[0].click();", next_btn)
time.sleep(random.uniform(3, 5))
# 等待新页面加载
try:
WebDriverWait(driver, 15).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, ".gl-item, .goods-list-v2 .item, .j-sku-item"))
)
print("翻页成功")
except TimeoutException:
print("翻页后商品加载超时，继续尝试...")
except (TimeoutException, NoSuchElementException):
print("无法找到下一页按钮，尝试URL翻页...")
current_url = driver.current_url
if "page=" in current_url:
new_page = page + 1
new_url = re.sub(r"page=\d+", f"page={new_page}", current_url)
else:
new_url = current_url + f"&page={new_page}"
driver.get(new_url)
time.sleep(random.uniform(3, 5))
print(f"已跳转到第 {new_page} 页")
except Exception as e:
print(f"爬取过程中发生严重错误: {str(e)}")
# 保存当前页面供调试
with open("jd_error_page.html", "w", encoding="utf-8") as f:
f.write(driver.page_source)
print("错误页面已保存为 jd_error_page.html")
finally:
driver.quit()
print(f"浏览器已关闭")
print(f"爬取完成，共获取 {len(books_data)} 条图书数据")
return books_data
# 其他函数保持不变（analyze_comments_sentiment, main等）
def analyze_comments_sentiment(comment_url):
"""评论情感分析"""
if not comment_url:
print("无有效URL，跳过评论分析")
return []
print(f"开始分析评论: {comment_url}")
driver = get_driver()
sentiments = []
try:
driver.get(comment_url)
time.sleep(random.uniform(3, 5))
print("商品页面加载成功")
# 切换到评论标签 - 更健壮的等待方式
print("切换到评论标签...")
try:
# 尝试点击评论标签
comment_tab = WebDriverWait(driver, 15).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "[data-anchor='#comment']"))
)
driver.execute_script("arguments[0].click();", comment_tab)
time.sleep(random.uniform(2, 3))
print("评论标签切换成功")
except:
# 如果找不到元素，尝试直接访问评论URL
if "#comment" not in driver.current_url:
driver.get(comment_url + "#comment")
print("直接访问评论页")
time.sleep(random.uniform(3, 5))
# 提取评论内容
comments = []
print("开始提取评论...")
for page_num in range(1, 4): # 最多尝试3页
print(f"处理评论页 {page_num}")
try:
# 等待评论加载
WebDriverWait(driver, 15).until(
EC.presence_of_element_located((By.CSS_SELECTOR, ".comment-item"))
)
time.sleep(random.uniform(1, 2))
soup = BeautifulSoup(driver.page_source, 'html.parser')
comment_items = soup.select('.comment-item')
print(f"本页找到 {len(comment_items)} 条评论")
for idx, item in enumerate(comment_items):
try:
comment_elem = item.select_one('.comment-con') or item.select_one('.comment-content')
if comment_elem:
comment = comment_elem.get_text(strip=True)
if 10 < len(comment) < 200: # 过滤过长/过短评论
comments.append(comment)
except:
continue
# 检查是否达到所需评论数
if len(comments) >= 15:
print(f"已收集足够评论({len(comments)}条)")
break
# 尝试翻页
try:
next_btn = driver.find_element(By.CSS_SELECTOR, '.ui-pager-next')
if "disabled" in next_btn.get_attribute("class"):
print("已是最后一页")
break
print("翻到下一页评论")
driver.execute_script("arguments[0].scrollIntoView();", next_btn)
time.sleep(0.5)
driver.execute_script("arguments[0].click();", next_btn)
time.sleep(random.uniform(2, 4))
except Exception as e:
print(f"评论翻页失败: {str(e)[:50]}...")
break
except Exception as e:
print(f"评论页处理错误: {str(e)[:50]}...")
break
except Exception as e:
print(f"评论爬取失败: {str(e)[:50]}...")
finally:
driver.quit()
if not comments:
print("未获取到有效评论")
return []
print(f"共获取 {len(comments)} 条评论，开始情感分析...")
# 情感分析
sentiment_results = []
for i, comment in enumerate(comments[:15]): # 限制分析数量
try:
# 控制请求频率
if i > 0 and i % 3 == 0:
delay = random.uniform(0.5, 1.5)
time.sleep(delay)
# 确保文本长度符合要求
text = comment[:min(len(comment), 500)]
result = client.sentimentClassify(text)
if 'items' in result and len(result['items']) > 0:
sentiment = result['items'][0]['sentiment']
sentiment_results.append(sentiment)
# 打印进度
if (i + 1) % 5 == 0:
print(f"已分析 {i + 1}/{len(comments[:15])} 条评论")
except Exception as e:
print(f"评论 {i + 1} 情感分析错误: {str(e)[:50]}...")
print(f"情感分析完成，共分析 {len(sentiment_results)} 条评论")
return sentiment_results
def main():
print("=" * 50)
print("京东图书爬虫与情感分析程序启动（支持登录）")
print("=" * 50)
# 创建浏览器实例
driver = get_driver()
# 执行登录
login_success = login_jd(driver)
# 爬取图书数据
print("\n开始爬取京东图书数据...")
books = jd_book_crawler("Python编程", max_pages=2, driver=driver)
# 保存结果到CSV
if books:
filename = 'jd_books.csv'
with open(filename, 'w', newline='', encoding='utf-8-sig') as f:
writer = csv.DictWriter(f, fieldnames=['title', 'price', 'isbn', 'url'])
writer.writeheader()
writer.writerows(books)
print(f"\n数据已保存到 {filename}")
# 对第一本书进行评论分析
if books:
first_book = books[0]
print(f"\n开始分析书籍评论: {first_book['title']}")
sentiments = analyze_comments_sentiment(first_book['url'])
if sentiments:
# 统计情感分布
positive = sum(1 for s in sentiments if s == 2)
negative = sum(1 for s in sentiments if s == 0)
neutral = sum(1 for s in sentiments if s == 1)
total = len(sentiments)
print("\n情感分析结果:")
print(f"样本数量: {total}")
print(f"积极评价m: {positive} ({positive / total:.1%})")
print(f"中性评价: {neutral} ({neutral / total:.1%})")
print(f"消极评价: {negative} ({negative / total:.1%})")
else:
print("未获取到有效评论数据")
else:
print("没有可分析的图书数据")
else:
print("未爬取到图书数据")
print("\n程序执行完成")
driver.quit()
if __name__ == "__main__":
main()
D:\python.exe C:/Users/32936/PycharmProjects/数据采集/作业.py
==================================================
京东图书爬虫与情感分析程序启动（支持登录）
==================================================
开始登录京东账号...
请手动完成登录...
等待用户登录（你有60秒时间完成登录）
Traceback (most recent call last):
File "C:\Users\32936\PycharmProjects\数据采集\作业.py", line 469, in <module>
main()
File "C:\Users\32936\PycharmProjects\数据采集\作业.py", line 427, in main
books = jd_book_crawler("Python编程", max_pages=2, driver=driver)
TypeError: jd_book_crawler() got an unexpected keyword argument 'driver'
登录超时，继续未登录状态爬取
开始爬取京东图书数据...
Process finished with exit code 1
在这个代码的基础上修改
京东商品详情数据接口提供了商品详细信息，如标题、价格等，适用于电商应用、数据分析、个性化推荐等。开发者需注册获取API权限，注意遵守规则和安全操作。
扫一扫
1342
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
