xiecheng_spider携程民宿爬虫

xiecheng_spider携程民宿爬虫

获取URL

由于携程网页由CSS编写，因此我们需要使用浏览器去跳转页面，这里我需要做一些准备工作

1.安装谷歌浏览器驱动，链接如下。安装成功后要放在PATH路径下

http://npm.taobao.org/mirrors/chromedriver/

2.安装环境，这里我用的Python3.7

# 我们需要使用其中的模块
pip install selenium

接下来我们来进行代码实现

首先引入一些需要的模块和html的URL

#!/usr/bin/env python
# coding=utf-8
import sys
import time
import json
import xlsxwriter
from selenium import webdriver

url = "https://inn.ctrip.com/onlineinn/list?checkin=2019-06-15&checkout=2019-06-16&cityid=5&cityname=%E5%93%88%E5%B0%94%E6%BB%A8&channelid=211"
# 定义一个字典稍后存储数据的时候用
items = []

接下来我们需要使用谷歌浏览器进行页面操作

#!/usr/bin/env python
# coding=utf-8
import sys
import time
import json
import xlsxwriter
from selenium import webdriver

url = "https://inn.ctrip.com/onlineinn/list?checkin=2019-06-15&checkout=2019-06-16&cityid=5&cityname=%E5%93%88%E5%B0%94%E6%BB%A8&channelid=211"
items = []

# 谷歌驱动
def chrome_driver(url):
    driver = webdriver.Chrome()
    driver.get(url)
    return driver

接下来我们了解一下Xpath

XPath即为XML路径语言（XML Path Language），它是一种用来确定XML文档中某部分位置的语言。

通过Xpath我们就能够获取相应的数据

#!/usr/bin/env python
# coding=utf-8
import sys
import time
import json
import xlsxwriter
from selenium import webdriver

url = "https://inn.ctrip.com/onlineinn/list?checkin=2019-06-15&checkout=2019-06-16&cityid=5&cityname=%E5%93%88%E5%B0%94%E6%BB%A8&channelid=211"
items = []

# 谷歌驱动
def chrome_driver(url):
    driver = webdriver.Chrome()
    driver.get(url)
    return driver

# 获取信息
def get_items(driver):
    time.sleep(1)
    # 展开全部
    show_all = driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[2]/div[3]/div[2]/div[4]')
    show_all.click()
    # 字典获取
    feed_item = {
   
   
        'info':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[1]/div/div[1]/div[1]').text,
        'type_of_house':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[2]/div[3]/div[1]/div[1]/span[1]').text,
        'num_of_people':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[2]/div[3]/div[1]/div[2]/span[1]').text,
        'num_of_bed':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[2]/div[3]/div[1]/div[3]/span[1]').text,
        'if_cook':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[2]/div[3]/div[1]/div[1]/span[3]').text,
        'if_swim':'No',
        'if_supermarket':'No',
        'locate':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[1]/div/div[1]/div[3]/span').text,
        'url':driver.current_url,
        'price':driver.find_element_by_class_name('txt-price').text
    }
    if driver.find_elements_by_class_name('item-device-0507') != []:
        feed_item['id_swim'] = 'Have swim pool'
    if driver.find_elements_by_class_name("item-device-0501") != []:
        feed_item['if_supermarket'] = "Have super market"
    items.append(feed_item) # 将数据保存到items字典中

由于我们找不到每个hotel的相应URL，因此我们需要浏览器进行虚拟操作来帮助我们

#!/usr/bin/env python
# coding=utf-8
import sys
import time
import json
import xlsxwriter
from selenium import webdriver

url = "https://inn.ctrip.com/onlineinn/list?checkin=2019-06-15&checkout=2019-06-16&cityid=5&cityname=%E5%93%88%E5%B0%94%E6%BB%A8&channelid=211"
items = []

# 谷歌驱动
def chrome_driver(url):
    driver = webdriver.Chrome()
    driver.get(url)
    return driver

# 获取信息
def get_items(driver):
    time.sleep(1)
    # 展开全部
    show_all = driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[2]/div[3]/div[2]/div[4]')
    show_all.click()
    # 字典获取
    feed_item = {
        'info':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[1]/div/div[1]/div[1]').text,
        'type_of_house':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[2]/div[3]/div[1]/div[1]/span[1]').text,
        'num_of_people':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[2]/div[3]/div[1]/div[2]/span[1]').text,
        'num_of_bed':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[2]/div[3]/div[1]/div[3]/span[1]').text,
        'if_cook':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[2]/div[3]/div[1]/div[1]/span[3]').text,
        'if_swim':'No',
        'if_supermarket':'No',
        'locate':driver.find_element_by_xpath('//*[@id="__next"]/div/div/div[1]/div/div[1]/div[3]/span').text,
        'url':driver.current_url,
        'price':driver.find_element_by_class_name('txt-price').text
    }
    if driver.find_elements_by_class_name('item-device-0507') != []:
        feed_item['id_swim'] = 'Have swim pool'
    if driver.find_elements_by_class_name("item-device-0501") != []:
        feed_item['if_supermarket'] = "Have super market"
    items.append(feed_item) # 将数据保存到items字典中

# 信息组合
def get_all_infos(driver):
    cnt = 0
    time.sleep(3)
    click_items = driver.find_elements_by_class_name("list-photo")
    for each in click_items:
        cnt = cnt + 1
        each.click() #浏览器跳转
        handle = driver.window_handles
        driver.switch_to.