Python selenium 爬取淘宝商品

Catalog

翻页

from urllib import parse
domain = 'https://s.taobao.com/search?'
keyword = 'Python3网络爬虫视频'
for i in range(199):
    # URL：页码+搜索词
    page = i * 44
    url = domain + parse.urlencode({'q': keyword, 's': page})
    print(url)

获取动态页面信息

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# 创建浏览器驱动+显式等待
driver = webdriver.Firefox()
driver.get('https://s.taobao.com/search?q=Python爬虫&s=44')
wait = WebDriverWait(driver, 9)
# 等待页面加载
wait.until(EC.presence_of_element_located(
    (By.CSS_SELECTOR, '#mainsrp-itemlist .items .item')))
# 页面源码
html = driver.page_source
print(html)

提取商品信息

from pyquery import PyQuery as pq
doc = pq(html)
# 获取当前页所有商品信息，返回生成器
items = doc('#mainsrp-itemlist .items .item').items()
for n, item in enumerate(items):
    product = {
        'image': item.find('.pic .img').attr('src'),
        'price': item.find('.price').text(),
        'deal_cnt': item.find('.deal-cnt').text()[:-3],
        'title': item.find('.title').text(),
        'shop': item.find('.shop').text(),
        'location': item.find('.location').text()}

完整代码

from urllib import parse
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from pyquery import PyQuery as pq

# 创建浏览器驱动+显式等待
driver = webdriver.Firefox()
wait = WebDriverWait(driver, 9)

# 获取商品信息
def get_products():
    # 等待页面加载
    wait.until(EC.presence_of_element_located(
        (By.CSS_SELECTOR, '#mainsrp-itemlist .items .item')))
    # 页面源码
    html = driver.page_source
    # 创建PyQuery对象
    doc = pq(html)
    # 获取当前页所有商品信息，返回生成器
    items = doc('#mainsrp-itemlist .items .item').items()
    for n, item in enumerate(items):
        product = {
            # 'image': item.find('.pic .img').attr('src'),
            'price': item.find('.price').text(),
            'deal_cnt': item.find('.deal-cnt').text()[:-3],
            'title': item.find('.title').text(),
            'shop': item.find('.shop').text(),
            'location': item.find('.location').text()}
        # 一页44个商品
        if n < 44:
            print(n + 1, product)
        else:
            break

# 淘宝搜索+翻页
def main():
    domain = 'https://s.taobao.com/search?'
    keyword = 'Python3网络爬虫视频'
    for i in range(199):
        # URL：页码+搜索词
        page = i * 44
        url = domain + parse.urlencode({'q': keyword, 's': page})
        driver.get(url)
        # 获取商品信息
        try:
            get_products()
        except TimeoutException as te:
            print('结束页：', url)
            break
    # 关闭浏览器
    driver.close()

if __name__ == '__main__':
    main()