python 爬取中国房价行情网网站

原创

已于 2023-07-30 14:46:55 修改 · 902 阅读

5 ·

CC 4.0 BY-SA版权

文章标签：

#python

于 2020-09-19 00:37:13 首次发布

本文介绍了如何使用Python编写爬虫程序，从中国房价行情网抓取房地产数据。通过提供的GitHub链接，可以获取名为HousePriceSpider.py的完整代码示例，帮助读者了解和学习网络爬虫的实现。

HousePriceSpider.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import time
from multiprocessing import Pool, Manager, cpu_count
from urllib.parse import urljoin
import numpy as np
import pandas as pd
import hashlib
from lxml import etree
from ChinaHousePrice.session import SessionWrapper
from datetime import datetime
from ChinaHousePrice.common import AreaCodeDecoder
import pymysql
from sqlalchemy import create_engine
from ChinaHousePrice import config
from sqlalchemy.sql import text


class Spider(object):
    def __init__(self, prov, city, url):
        self.prov = prov
        self.city = city
        self.url = url
        self.__session = SessionWrapper(timeout=20)
        self.__href_d = self._district()
        self.city_data = self._city_data()

    def _district(self):
        href_d = {}
        res = self.__session.get(self.url)
        if res is None:
            return href_d
        html = etree.HTML(res.text)
        href_n = html.xpath("//span[@class='city-n']/a/@href")
        district_name_n = html.xpath("//span[@class='city-n']/a/span/text()")
        href_d.update(dict(zip(district_name_n, href_n)))
        href_w = html.xpath("//span[@class='city-w']/a/@href")
        district_name_w = html.xpath("//span[@class='city-w']/a/span/text()")
        href_d.update(dict(zip(district_name_w, href_w)))
        return href_d

    def _city_data(self):
        return self._district_all_data(np.nan, self.url + "?")

    def _district_all_data(self, district, url):
        params = ["", "?&type=newha", "?&type=lease", "?&type=lease&proptype=22", "?&proptype=22", "?&proptype=21",
                  "?&type=lease&proptype=22"]
        result = [self.prov, self.city, district]
        for u in params:
            url_ = url + u
            result += self._parse_by_xp