2021-01-04

Python爬取链家南京租房信息

接上个文章重新写了遍这个爬虫，大半年过去了都
不会写的兄弟萌看我上个文章嗷
这次写的时候发现在style那边如果写style = mix[3].strip()会报IndexError: list index out of range这个错
这边直接style = mix[-1].strip()这样就行了
有无大佬能告诉我怎么改能直接一个代码就把这几个区的信息都搞完，我是新手属实不太行

import time, re, csv, requests
import codecs
from bs4 import BeautifulSoup

with open(r'C:\Users\ZXC\Desktop\链家\鼓楼.csv', 'ab+')as fp:
    fp.write(codecs.BOM_UTF8)
f = open(r'C:\Users\ZXC\Desktop\链家\鼓楼.csv','a+',newline='', encoding='utf-8')
writer = csv.writer(f)
writer.writerow(('名称', '租金', '面积', '具体地址', '门朝向', '户型', '地区'))
r'C:\Users\ZXC\Desktop\链家\鼓楼.csv'

"https://nj.lianjia.com/zufang/gulou/pg3/#contentList"
urls = ['https://nj.lianjia.com/zufang/gulou/pg{}/'.format(str(i)) for i in range(101)]
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.9 Safari/537.36'
}
for url in urls:
    res = requests.get(url, headers=headers)
    # print(res.text) 网页内容 文本
    # print(res.content.decode('utf-8'))  # 网页内容 二进制
    content = res.text
    soup = BeautifulSoup(content, 'html.parser')
    # print(soup)
    infos = soup.find('div',{'class':'content__list'}).find_all('div',{'class':'content__list--item'})
    # print(infos)
    for info in infos:
        title = info.find('p', {'class': 'content__list--item--title'}).find('a').get_text().strip()
        price = info.find('span', {'class': 'content__list--item-price'}).get_text()
        mixed = info.find('p', {'class': 'content__list--item--des'}).get_text()
        mix = re.split(r'/', mixed)
        address = mix[0].strip()
        area = mix[1].strip()
        door_orientation = mix[2].strip()
        style = mix[-1].strip()
        region = re.split(r'-', address)[0]
        writer.writerow((title, price, area, address, door_orientation, style, region))
        time.sleep(0)

f.close()