我爱我家-北京-mysql-优快云博客

本文介绍了一个使用Python的requests库和lxml库抓取我爱我家网站上北京回龙观地区租房信息的方法。通过解析网页源码，提取了房源标题、详细描述等关键信息，并将这些数据插入到MySQL数据库中。

import requests
from lxml import etree
from day3.mysql_test import mysql_conn

myhelper = mysql_conn()
sql = 'insert into woaiwojia(target,title,title_1,title_3,strong) values (%s, %s, %s ,%s, %s)'




headers = {
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
}

for i in range(1,4):
    url ='https://bj.5i5j.com/zufang/huilongguan/n{}/'.format(i)
    # print(url)
    response = requests.get(url, headers=headers)

    # with open('woaiwojia.html', 'wb') as f:
    #     f.write(response.content)

    html_ele = etree.HTML(response.text)

    # print(html_ele)
    li_list = html_ele.xpath('//ul[@class="pList"]/li')
    # print(li_list)
    for li_ele in li_list:
        # print(li_ele)
        target = li_ele.xpath('./div[2]/h3/a')[0].text
        # print(target)
        title = li_ele.xpath('./div[2]/div[1]/p[1]/text()')[0]
        # print(title)
        title_1 = li_ele.xpath('./div[2]/div[1]/p[2]/a/text()')[0] + li_ele.xpath('./div[2]/div[1]/p[2]/text()')[0]
        # print(title_1)
        title_2 = li_ele.xpath('./div[2]/div[2]/span/text()')
        for v in title_2:
            title_3 = v
            # print(v)
            strong = li_ele.xpath('./div[2]/div[1]/div[1]/p/strong')[0].text
            # print(strong)
            data = (target,title,title_1,title_3,strong)

            myhelper.execute_modify_mysql(sql, data)

转载于:https://www.cnblogs.com/gxsmm/p/9496844.html