美团数城商铺榜

# -*- coding:utf-8 -*-
# 仅需修改这个地方https://jn.lianjia.com/ershoufang/pg{}rs/   将jn换成你所在城市的拼写首字母小写
import requests
from lxml import etree
import time
import random
import csv
import requests
import json


class LianjiaSpider(object):
    def __init__(self):
        self.url = "https://mobilenext-web.meituan.com/api/newSalesBoard/getSaleBoardDetail?cityId={}&boardType=8&districtId=0&cateId=1&offset=0&limit=50"

        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1"}

    def get_page(self, url):
        res = requests.get(url=url, headers=self.headers)
        res.encoding = "utf-8"
        html = res.text
        # print(html)

        #  results_temp = html.replace('{"totalSize":50,"saleBoardDealList":', "").replace("}}]}", "")
        # results = results_temp + "}}]"
        #  print(results)

        self.parse_page(html)

    # print(html)
    # print(i)
    def parse_page(self, html):
        results = html[35:-20]
        print(results)
        for list in json.loads(results):
            # print(list)
            id = list["id"]
            name = list["name"]
            weekSaleCount = list["weekSaleCount"]
            score = list["score"]
            avgPrice = list["avgPrice"]
            cateName = list["cateName"]
            areaName = list["areaName"]
            distance = list["distance"]
            rank = list["rank"]
            frontImg = list["frontImg"]
            oneSentence = list["oneSentence"]
            # SaleBoardPoiGroup = list["SaleBoardPoiGroup"]
            #  saleBoardPoiCoupon = list["saleBoardPoiCoupon"]
            #   saleBoardPoiPay = list["saleBoardPoiPay"]
            # banchList = list["banchList"]
            with open('meituan.csv', 'a', newline='', encoding='utf-8')as f:
                write = csv.writer(f)
                # write.writerow([id,name,weekSaleCount,score,avgPrice,cateName,areaName,distance,rank,frontImg,oneSentence,SaleBoardPoiGroup,saleBoardPoiCoupon,saleBoardPoiPay,banchList])
                write.writerow([id, name, weekSaleCount, score, avgPrice, cateName, areaName, distance, rank, frontImg,
                                oneSentence])

                f.close()

    def main(self):
        for i in range(88, 99):
            time.sleep(random.randint(3, 5))
            url = self.url.format(i)
            #  print(url)
            self.get_page(url)
        # print(j)


if __name__ == '__main__':
    start = time.time()
    spider = LianjiaSpider()
    spider.main()
    end = time.time()
    print("执行时间：%.2f" % (end - start))