from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import TimeoutException, NoSuchElementException, WebDriverException
from bs4 import BeautifulSoup
import requests
import re
import time
import csv
import random
import os
import sys
from aip import AipNlp
from webdriver_manager.chrome import ChromeDriverManager
# 百度AI配置
APP_ID = '119348823'
API_KEY = 'BMUyFD1qn0p4BgaRL5ZsFAHS'
SECRET_KEY = 'jSvSCxAOm47OGB7JxM0g05UKWeagFFPe'
client = AipNlp(APP_ID, API_KEY, SECRET_KEY)
# 随机请求头
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.142 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15"
]
def get_driver():
"""创建并配置浏览器驱动"""
chrome_options = Options()
# 调试时注释掉无头模式
# chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--window-size=1920,1080")
chrome_options.add_argument(f"user-agent={random.choice(USER_AGENTS)}")
chrome_options.add_argument("--disable-blink-features=AutomationControlled")
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)
# 自动下载并管理ChromeDriver
service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=service, options=chrome_options)
# 隐藏自动化特征
driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
return driver
def jd_book_crawler(search_keyword, max_pages=2):
"""京东图书爬虫函数 - 增强健壮性"""
print("启动浏览器...")
driver = get_driver()
books_data = []
try:
print(f"访问京东首页...")
driver.get("https://www.jd.com")
time.sleep(random.uniform(1, 3))
print("京东首页加载成功")
# 搜索图书
print(f"搜索关键词: {search_keyword}")
try:
# 多种定位搜索框的方式
search_box = WebDriverWait(driver, 15).until(
EC.element_to_be_clickable((By.ID, "key"))
)
except TimeoutException:
# 备选定位方式
search_box = WebDriverWait(driver, 15).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "input.search-key"))
)
# 模拟真实输入
search_box.clear()
for char in search_keyword:
search_box.send_keys(char)
time.sleep(random.uniform(0.05, 0.15))
# 提交搜索
search_box.send_keys(Keys.ENTER)
print("已提交搜索")
time.sleep(random.uniform(2, 4))
# 验证是否跳转到搜索结果页
try:
WebDriverWait(driver, 15).until(
EC.url_contains("search")
)
print("已进入搜索结果页")
except TimeoutException:
print("可能遇到验证码或反爬页面，尝试重新加载...")
driver.get(f"https://search.jd.com/Search?keyword={search_keyword}")
time.sleep(random.uniform(3, 5))
# 等待结果加载
print("等待商品列表加载...")
try:
# 多种等待商品加载的方式
WebDriverWait(driver, 20).until(
EC.presence_of_element_located((By.CSS_SELECTOR, ".gl-item, .goods-list-v2 .item, .j-sku-item"))
)
time.sleep(random.uniform(1, 2))
except TimeoutException:
print("商品加载超时，尝试备用加载方式...")
# 尝试滚动页面触发加载
driver.execute_script("window.scrollTo(0, document.body.scrollHeight/3);")
time.sleep(2)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight/2);")
time.sleep(2)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(3)
for page in range(1, max_pages + 1):
print(f"正在处理第 {page} 页")
# 获取页面源码
html = driver.page_source
soup = BeautifulSoup(html, 'html.parser')
# 多种商品列表选择器
items = soup.select('.gl-item') # 京东主站
if not items:
items = soup.select('.goods-list-v2 .item') # 图书频道
if not items:
items = soup.select('.j-sku-item') # 备用选择器
if not items:
print("警告：未找到商品元素，尝试保存页面源码以供分析")
with open(f"jd_page_{page}.html", "w", encoding="utf-8") as f:
f.write(html)
print("页面源码已保存")
continue
print(f"本页找到 {len(items)} 个商品")
for idx, item in enumerate(items):
try:
# 多种标题选择器
title_elem = item.select_one('.p-name a em') or \
item.select_one('.p-name a') or \
item.select_one('.p-name-type-2 a') or \
item.select_one('.name a')
title = title_elem.text.strip() if title_elem else "N/A"
# 多种价格选择器
price_elem = item.select_one('.p-price strong') or \
item.select_one('.p-price i') or \
item.select_one('.price-box .price') or \
item.select_one('.j-price')
price = price_elem.text.strip() if price_elem else "0.00"
# 提取SKU/ISBN
isbn = item.get('data-sku') or item.get('data-spu') or "N/A"
# 详情页URL
detail_elem = item.select_one('.p-img a') or \
item.select_one('.pic a') or \
item.select_one('.name a')
detail_url = detail_elem['href'] if detail_elem and 'href' in detail_elem.attrs else ""
if detail_url and not detail_url.startswith('http'):
detail_url = 'https:' + detail_url
books_data.append({
'title': title,
'price': price,
'isbn': isbn,
'url': detail_url
})
if (idx + 1) % 5 == 0:
print(f"已处理 {idx + 1}/{len(items)} 个商品")
except Exception as e:
print(f"商品 {idx + 1} 提取错误: {str(e)[:50]}...")
# 翻页处理
if page < max_pages:
print("尝试翻页...")
try:
# 多种翻页按钮定位方式
next_btn = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, '.pn-next, .pn-next:not(.disabled)'))
)
driver.execute_script("arguments[0].scrollIntoView({behavior: 'smooth', block: 'center'});",
next_btn)
time.sleep(0.5)
driver.execute_script("arguments[0].click();", next_btn)
time.sleep(random.uniform(3, 5))
# 等待新页面加载
try:
WebDriverWait(driver, 15).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, ".gl-item, .goods-list-v2 .item, .j-sku-item"))
)
print("翻页成功")
except TimeoutException:
print("翻页后商品加载超时，继续尝试...")
except (TimeoutException, NoSuchElementException):
print("无法找到下一页按钮，尝试URL翻页...")
current_url = driver.current_url
if "page=" in current_url:
new_page = page + 1
new_url = re.sub(r"page=\d+", f"page={new_page}", current_url)
else:
new_url = current_url + f"&page={new_page}"
driver.get(new_url)
time.sleep(random.uniform(3, 5))
print(f"已跳转到第 {new_page} 页")
except Exception as e:
print(f"爬取过程中发生严重错误: {str(e)}")
# 保存当前页面供调试
with open("jd_error_page.html", "w", encoding="utf-8") as f:
f.write(driver.page_source)
print("错误页面已保存为 jd_error_page.html")
finally:
driver.quit()
print(f"浏览器已关闭")
print(f"爬取完成，共获取 {len(books_data)} 条图书数据")
return books_data
# 其他函数保持不变（analyze_comments_sentiment, main等）
def analyze_comments_sentiment(comment_url):
"""评论情感分析"""
if not comment_url:
print("无有效URL，跳过评论分析")
return []
print(f"开始分析评论: {comment_url}")
driver = get_driver()
sentiments = []
try:
driver.get(comment_url)
time.sleep(random.uniform(3, 5))
print("商品页面加载成功")
# 切换到评论标签 - 更健壮的等待方式
print("切换到评论标签...")
try:
# 尝试点击评论标签
comment_tab = WebDriverWait(driver, 15).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "[data-anchor='#comment']"))
)
driver.execute_script("arguments[0].click();", comment_tab)
time.sleep(random.uniform(2, 3))
print("评论标签切换成功")
except:
# 如果找不到元素，尝试直接访问评论URL
if "#comment" not in driver.current_url:
driver.get(comment_url + "#comment")
print("直接访问评论页")
time.sleep(random.uniform(3, 5))
# 提取评论内容
comments = []
print("开始提取评论...")
for page_num in range(1, 4): # 最多尝试3页
print(f"处理评论页 {page_num}")
try:
# 等待评论加载
WebDriverWait(driver, 15).until(
EC.presence_of_element_located((By.CSS_SELECTOR, ".comment-item"))
)
time.sleep(random.uniform(1, 2))
soup = BeautifulSoup(driver.page_source, 'html.parser')
comment_items = soup.select('.comment-item')
print(f"本页找到 {len(comment_items)} 条评论")
for idx, item in enumerate(comment_items):
try:
comment_elem = item.select_one('.comment-con') or item.select_one('.comment-content')
if comment_elem:
comment = comment_elem.get_text(strip=True)
if 10 < len(comment) < 200: # 过滤过长/过短评论
comments.append(comment)
except:
continue
# 检查是否达到所需评论数
if len(comments) >= 15:
print(f"已收集足够评论({len(comments)}条)")
break
# 尝试翻页
try:
next_btn = driver.find_element(By.CSS_SELECTOR, '.ui-pager-next')
if "disabled" in next_btn.get_attribute("class"):
print("已是最后一页")
break
print("翻到下一页评论")
driver.execute_script("arguments[0].scrollIntoView();", next_btn)
time.sleep(0.5)
driver.execute_script("arguments[0].click();", next_btn)
time.sleep(random.uniform(2, 4))
except Exception as e:
print(f"评论翻页失败: {str(e)[:50]}...")
break
except Exception as e:
print(f"评论页处理错误: {str(e)[:50]}...")
break
except Exception as e:
print(f"评论爬取失败: {str(e)[:50]}...")
finally:
driver.quit()
if not comments:
print("未获取到有效评论")
return []
print(f"共获取 {len(comments)} 条评论，开始情感分析...")
# 情感分析
sentiment_results = []
for i, comment in enumerate(comments[:15]): # 限制分析数量
try:
# 控制请求频率
if i > 0 and i % 3 == 0:
delay = random.uniform(0.5, 1.5)
time.sleep(delay)
# 确保文本长度符合要求
text = comment[:min(len(comment), 500)]
result = client.sentimentClassify(text)
if 'items' in result and len(result['items']) > 0:
sentiment = result['items'][0]['sentiment']
sentiment_results.append(sentiment)
# 打印进度
if (i + 1) % 5 == 0:
print(f"已分析 {i + 1}/{len(comments[:15])} 条评论")
except Exception as e:
print(f"评论 {i + 1} 情感分析错误: {str(e)[:50]}...")
print(f"情感分析完成，共分析 {len(sentiment_results)} 条评论")
return sentiment_results
def main():
print("=" * 50)
print("京东图书爬虫与情感分析程序启动")
print("=" * 50)
# 爬取图书数据
print("\n开始爬取京东图书数据...")
books = jd_book_crawler("Python编程", max_pages=2)
# 保存结果到CSV
if books:
filename = 'jd_books.csv'
with open(filename, 'w', newline='', encoding='utf-8-sig') as f:
writer = csv.DictWriter(f, fieldnames=['title', 'price', 'isbn', 'url'])
writer.writeheader()
writer.writerows(books)
print(f"\n数据已保存到 {filename}")
# 对第一本书进行评论分析
if books:
first_book = books[0]
print(f"\n开始分析书籍评论: {first_book['title']}")
sentiments = analyze_comments_sentiment(first_book['url'])
if sentiments:
# 统计情感分布
positive = sum(1 for s in sentiments if s == 2)
negative = sum(1 for s in sentiments if s == 0)
neutral = sum(1 for s in sentiments if s == 1)
total = len(sentiments)
print("\n情感分析结果:")
print(f"样本数量: {total}")
print(f"积极评价: {positive} ({positive / total:.1%})")
print(f"中性评价: {neutral} ({neutral / total:.1%})")
print(f"消极评价: {negative} ({negative / total:.1%})")
else:
print("未获取到有效评论数据")
else:
print("没有可分析的图书数据")
else:
print("未爬取到图书数据")
print("\n程序执行完成")
if __name__ == "__main__":
main()
这个代码中有需要在开发者模式中获取实际数据吗，怎样获取怎样替换呢
最新发布
穿云API能帮助程序员绕过验证码验证，包括Anti-bot检查、Cloudflare和CAPTCHA，提高开发效率，降低自动化操作的障碍。
扫一扫
1万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
