成功输出美团app美食销售人气榜网页

最新推荐文章于 2023-07-26 11:18:03 发布

原创最新推荐文章于 2023-07-26 11:18:03 发布 · 175 阅读

CC 4.0 BY-SA版权

本文介绍如何使用Python爬虫技术从链家网站获取房产信息，包括房价、单价、房屋详情和位置，实现批量数据抓取并保存至CSV文件，适合房产数据分析初学者和开发者参考。

# 仅需修改这个地方https://jn.lianjia.com/ershoufang/pg{}rs/   将jn换成你所在城市的拼写首字母小写
import requests
from lxml import etree
import time
import random
import csv


class LianjiaSpider(object):
    def __init__(self):
        self.url = "https://mobilenext-web.meituan.com/newSalesBoard?boardType=8&ci=96&areaId=6723&cateId=2&districtId=0&poiId=595175031&f=android&lat=36.526046191159445&lng=122.062217811"

        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1"}

    def get_page(self, url):
        res = requests.get(url=url, headers=self.headers)
        res.encoding = "utf-8"
        html = res.text
        print(html)
        # 调用解析函数
     #   self.parse_page(html)

    def parse_page(self, html):
        parse_html = etree.HTML(html)
        house_dict = {}
        house_list = parse_html.xpath("//div//ul[@class='sellListContent']//li")
        for house in house_list:
            house_dict["name"] = house.xpath(".//div[@class='title']/a//text()")[0]
            house_dict["totalprice"] = house.xpath(".//div[@class='totalPrice']/span//text()")[0]
            house_dict["uniteprice"] = house.xpath(".//div[@class='unitPrice']/span//text()")[0]
            house_dict["houseInfo"] = house.xpath(".//div[@class='houseInfo']/text()")[0]
            house_dict["positionInfoShequ"] = house.xpath(".//div[@class='positionInfo']/a[1]//text()")[0]
            house_dict["positionInfoJiedao"] = house.xpath(".//div[@class='positionInfo']/a[2]//text()")[0]
            house_dict["followInfo"] = house.xpath(".//div[@class='followInfo']/text()")[0]
            print(house_dict)
            with open('F:/top250/lianjia.csv', 'a', newline='', encoding='utf-8')as f:
                write = csv.writer(f)
                # write.writerow([house_dict["name"],house_dict["totalprice"],house_dict["uniteprice"],house_dict["info"]])
                write.writerow(
                    [house_dict["name"], house_dict["totalprice"], house_dict["uniteprice"], house_dict["houseInfo"],
                     house_dict["positionInfoShequ"], house_dict["positionInfoJiedao"], house_dict["followInfo"]])
            f.close()

    def main(self):
        try:
            for i in range(1, 2000):
                time.sleep(random.randint(3, 5))
                url = self.url.format(i)
                self.get_page(url)
        except:
            self.main()


if __name__ == '__main__':
    start = time.time()
    spider = LianjiaSpider()
    spider.main()
    end = time.time()
    print("执行时间：%.2f" % (end - start))