Day 6作业

最新推荐文章于 2025-08-23 20:28:56 发布

转载最新推荐文章于 2025-08-23 20:28:56 发布 · 94 阅读

0 ·

CC 4.0 BY-SA版权

原文链接：http://www.cnblogs.com/-zcj/p/11048722.html

文章标签：

#python #php

本次作业是爬取京东商品信息，如搜索凤爪商品信息。学习内容包括selenium选择器之Xpath、元素交互操作、破解登录，还使用selenium爬取了一页京东商品信息。总结称selenium用法多样，比request模块更方便，期待后续破解滑动验证。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

这次作业：爬取京东商品信息

from selenium import  webdriver
from selenium.webdriver.common.keys import Keys
import  time
driver = webdriver.Chrome()
def get_good(driver):
    try:

        #通过js控制滚轮滑动获取所有商品信息
        js_code = '''
            window.scrollTo(0,5000);
            '''
        driver.execute_script(js_code)#执行js代码


        #等待数据加载
        time.sleep(2)


        good_list = driver.find_elements_by_class_name('gl-item')

        for good in good_list:
            good_url = good.find_element_by_css_selector('.p-img a').get_attribute('href')
            good_name = good.find_element_by_css_selector('.p-name em').text.replace("\n", "--")
            good_price = good.find_element_by_class_name('p-price').text.replace("\n", ":")
            good_commit = good.find_element_by_class_name('p-commit').text.replace("\n", " ")


            good_content = f'''
                            商品连接:{good_url}
                            商品名称:{good_name}
                            商品价格:{good_price}
                            商品人数:{good_commit}
        
                            \n
    
                            '''
            print(good_content)
            with open('jd扩展.txt', 'a', encoding='utf-8')as f:
                f.write(good_content)


        next_tag = driver.find_element_by_class_name('pn-next')
        next_tag.click()
        time.sleep(2)

        #递归调用函数
        get_good(driver)

        time.sleep(10)
    finally:
        driver.close()

if __name__ == '__main__':
    good_name = input('请输入爬取商品信息：').strip()

    driver = webdriver.Chrome()
    driver.implicitly_wait(10)
    #1.
    driver.get('https://www.jd.com/')

    input_tag = driver.find_element_by_id('key')
    input_tag.send_keys(good_name)
    input_tag.send_keys(Keys.ENTER)

   get_good(driver)

结果部分截图
搜索商品信息凤爪

以下为今天所学

1.selenium选择器之Xpath

# from selenium import webdriver
#
# driver = webdriver.Chrome()
from selenium import webdriver

driver = webdriver.Chrome()
try:
    #隐式等待：写在get请求前
    driver.implicitly_wait(5)

    driver.get('https://doc.scrapy.org/en/latest/_static/selectors-sample1.html')
#显式等待：写在get请求后
#wait.until(...)
'''
    <html>
     <head>
      <base href='http://example.com/' />
      <title>Example website</title>
     </head>
     <body>
      <div id='images'>
       <a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' /></a>
       <a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
       <a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
       <a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
       <a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
      </div>
     </body>
    </html>
    '''
    # 根据Xpath语法查找元素
    #/从根节点开始找第一个
    html = driver.find_element_by_xpath('/html')
    #html = driver.find_element_by_Xpath('/head')#报错
    print(html.tag_name)

    #//从根节点开始找任意一个节点
    div = driver.find_element_by_xpath('//div')
    print(div.tag_name)

    #@
    #查找id为images的div节点
    div = driver.find_element_by_xpath('//div[@id="image"]')
    print(div.tag_name)
    print(div.text)

    #找到第一个a节点
    a = driver.find_element_by_xpath('//a')
    print(a.tag_name)

    #找到所有a节点
    a_s = driver.find_elements_by_xpath('//a')
    print(a_s)


    #找到第一个a节点的href属性
    #get_attribute：获取节点中某个属性
    a = driver.find_element_by_xpath('//a').get_attribute('href')
    print(a)

finally:
    driver.close()

2.selenium剩余操作（元素交互操作）

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time


driver = webdriver.Chrome()

'''
点击、清除操作
'''
try:
    driver.implicitly_wait(10)
    #1.往jd发送请求
    driver.get('https://www.jd.com/')
    #找到输入框输入围城
    input_tag = driver.find_element_by_id('key')
    input_tag.send_keys('围城')
    #键盘回车
    input_tag.send_keys(Keys.ENTER)
    time.sleep(2)
    #找到输入框输入墨菲定律
    input_tag = driver.find_element_by_id('key')
    input_tag.clear()
    input_tag.send_keys("墨菲定律")
    #找到搜索按钮点击搜索
    button = driver.find_element_by_class_name('button')
    button.click()
    time.sleep(10)
finally:
    driver.close()


'''
获取cookies

'''
from selenium import  webdriver
import time

driver = webdriver.chrome()

try:
    driver.implicitly_wait(10)
    driver.get('https://www.zhihu.com/explore')
    print(driver.get_cookies())

    time.sleep(10)

finally:
    driver.close()


'''
选项卡
'''

# 选项卡管理：切换选项卡，有js的方式windows.open,有windows快捷键：
# ctrl+t等，最通用的就是js的方式
import time
from selenium import webdriver

browser = webdriver.Chrome()
try:
    browser.get('https://www.baidu.com')

    # execute_script: 执行javascrpit代码
    # 弹窗操作
    # browser.execute_script('alert("tank")')
    # 新建浏览器窗口
    browser.execute_script(
        '''
        window.open();
        '''
    )
    time.sleep(1)
    print(browser.window_handles)  # 获取所有的选项卡
    # 切换到第二个窗口
    # 新:
    browser.switch_to.window(browser.window_handles[1])
    # 旧:
    # browser.switch_to_window(browser.window_handles[1])

    # 第二个窗口往淘宝发送请求
    browser.get('https://www.taobao.com')
    time.sleep(5)

    # 切换到第一个窗口
    browser.switch_to_window(browser.window_handles[0])
    browser.get('https://www.sina.com.cn')

    time.sleep(10)
finally:
    browser.close()

'''
ActionChanges动作链

'''
from selenium import webdriver
from selenium.webdriver import ActionChains
import time

driver = webdriver.Chrome()
driver.implicitly_wait(10)
driver.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')

try:

    # driver.switch_to_frame('iframeResult')
    # 切换到id为iframeResult的窗口内
    driver.switch_to.frame('iframeResult')

    # 源位置
    draggable = driver.find_element_by_id('draggable')

    # 目标位置
    droppable = driver.find_element_by_id('droppable')

    # 调用ActionChains，必须把驱动对象传进去
    # 得到一个动作链对象，赋值给一个变量
    actions = ActionChains(driver)

    # 方式一: 机器人
    # 瞬间把源图片位置秒移到目标图片位置
    # actions.drag_and_drop(draggable, droppable)  # 编写一个行为
    # actions.perform()  # 执行编写好的行为


    # 方式二: 模拟人的行为
    source = draggable.location['x']
    target = droppable.location['x']
    print(source, target)

    distance = target - source
    print(distance)

    # perform：每个动作都要调用perform执行

    # 点击并摁住源图片
    ActionChains(driver).click_and_hold(draggable).perform()

    s = 0
    while s < distance:
        # 执行位移操作
        ActionChains(driver).move_by_offset(xoffset=2, yoffset=0).perform()
        s += 2

    # 释放动作链
    ActionChains(driver).release().perform()

    time.sleep(10)


finally:
    driver.close()


'''
前进、后退
'''
from selenium import webdriver
import time

driver = webdriver.Chrome()


try:
    driver.implicitly_wait(10)
    driver.get('[图片]https://www.jd.com/')
    driver.get('https://www.baidu.com/')
    driver.get('https://www.cnblogs.com/')

    time.sleep(2)

    # 回退操作
    driver.back()
    time.sleep(1)
    # 前进操作
    driver.forward()
    time.sleep(1)
    driver.back()
    time.sleep(10)

finally:
    driver.close()

3.破解登录

from selenium import webdriver
from selenium.webdriver import ChromeOptions
import time
r'''
步骤：
    1.打开文件的查看，显示隐藏文件
    2.找到C:\Users\社会我静哥\AppData\Local\Google\Chrome\User Data
      删除Default文件
    3.重新打开浏览器，并登了百度账号
        -此时会创建一个新的Default缓存文件
'''

#获取options对象
options = ChromeOptions()
#获取cookies保存路径
# 'C:\Users\社会我静哥\AppData\Local\Google\Chrome\User Data'
profile_directory = r'--user-data-dir=C:\Users\社会我静哥\AppData\Local\Google\Chrome\User Data'
# 添加用户信息目录
options.add_argument(profile_directory)

# 把参数加载到当前驱动中  chrome_options默认参数，用来接收options对象
driver = webdriver.Chrome(chrome_options=options)

try:
    driver.implicitly_wait(10)
    driver.get('https://www.baidu.com/')
    '''
    BDUSS:FmS2FEZ1JWdVZqUG15YlZ-YnA1cUkwVXR-NGN3Q2laa3NTRzVCNmdBa3YzeTlkRVFBQUFBJCQAAAAAAAAAAAEAAAC7LAGbbG92ZXN1bW1lcjkyNQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC9SCF0vUghdZW
    '''
    # 添加用户cookies信息
    # name、value必须小写
    driver.add_cookie({"name": "BDUSS", "value": "FmS2FEZ1JWdVZqUG15YlZ-YnA1cUkwVXR-NGN3Q2laa3NTRzVCNmdBa3YzeTlkRVFBQUFBJCQAAAAAAAAAAAEAAAC7LAGbbG92ZXN1bW1lcjkyNQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC9SCF0vUghdZW"})

    # 刷新操作
    driver.refresh()

    time.sleep(20)

finally:
    driver.close()

4.selenium爬取一页京东商品信息

from selenium import  webdriver
from selenium.webdriver.common.keys import Keys
import  time
driver = webdriver.Chrome()
try:
    driver.implicitly_wait(10)

    driver.get('https://www.jd.com/')

    input_tag = driver.find_element_by_id('key')
    input_tag.send_keys('macbook')
    input_tag.send_keys(Keys.ENTER)

    good_list = driver.find_elements_by_class_name('gl-item')

    for good in good_list:

        good_url = good.find_element_by_css_selector('.p-img a').get_attribute('href')
        good_name = good.find_element_by_css_selector('.p-name em').text.replace("\n","--")
        good_price= good.find_element_by_class_name('p-price').text.replace("\n",":")
        good_commit = good.find_element_by_class_name('p-commit').text.replace("\n"," ")
        good_from = good.find_element_by_class_name('J_im_icon').text.replace("\n"," ")

        good_content = f'''
                        商品连接:{good_url}
                        商品名称:{good_name}
                        商品价格:{good_price}
                        商品人数:{good_commit}
                        商品商家:{good_from}
                        \n

                        '''
        print(good_content)
        with open('jd.txt','a',encoding='utf-8')as f:
                f.write(good_content)
    time.sleep(10)
finally:
    driver.close()