python 爬取北京新发地。

代码：

import csv

import requests
from lxml import etree
from concurrent.futures import ThreadPoolExecutor
headers={
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
    ,'Referer': 'http://www.xinfadi.com.cn/priceDetail.html',
    'X-Requested-With': 'XMLHttpRequest',
    'Cookie': 'SHOP_MANAGE=71a6170c-f8e0-432a-b329-e866610a16c0'

}


f= open('北京新发地菜价.csv', mode='a', encoding='utf-8', newline='')
wirter = csv.writer(f)

def download(url,pageNo):
        data = {
            "limit": 20,
            "current": pageNo,
            # "pubDateStartTime":"",
            # "pubDateEndTime":"",
            # "prodPcatid":"",
            # "prodCatid":"",
            # 'prodName':""
        }
        resp = requests.post(url,data=data)
        infomation=resp.json()['list']
        for i in infomation:
            peodname=i['prodName']
            avgprice=i['avgPrice']
            place=i['place']
            lowprice=i['lowPrice']
            highprice=i['highPrice']
            pubdate=i['pubDate']
            wirter.writerow([peodname,lowprice,highprice,avgprice,place,pubdate])

        print('over!')
        f.close()

    

if __name__ == '__main__':
    url="http://www.xinfadi.com.cn/getPriceData.html"
    with ThreadPoolExecutor(10000) as t:
         for i in range(1,30):
             t.submit(download,url,pageNo=i)

爬取的相应结果。