xpath,requests爬取小猪短租网

import requests
from lxml import etree
import time
headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) '
    'AppleWebKit/537.36 (KHTML, like Gecko) '
    'Chrome/70.0.3538.77 Safari/537.36'
}

def page_url(url):
    res = requests.get(url,headers=headers)
    text = res.text
    html = etree.HTML(text)
    links = html.xpath('.//a[@class="resule_img_a"]/@href')
    for link in links:
        parse_page(link)

def parse_page(url):
    res = requests.get(url,headers=headers)
    text = res.text
    html = etree.HTML(text)
    titles = html.xpath('.//div[@class="pho_info"]/h4/em//text()')
    addresses = html.xpath('.//span[@class="pr5"]//text()')
    prices = html.xpath('.//div[@class="day_l"]/span//text()')
    imgs = html.xpath('.//div[@class="member_pic"]/a//img/@src')
    names = html.xpath('.//div[@class="w_240"]/h6/a//text()')
    for title,address,price,img,name in zip(titles,addresses,prices,imgs,names):
        data ={
            '屋子标题':title.strip(),
            '地址':address.strip(),
            '价格':price,
            '图片':img,
            '名字':name
        }
        print(data)



if __name__ == '__main__':
    urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format
            (x)  for x in range(1,14)]
    for url in urls:
        page_url(url)
        time.sleep(2)