高德找房——爬取房源

柠七ζั͡ޓއއއ๓

已于 2024-06-22 19:51:51 修改

阅读量357

点赞数 3

文章标签： python beautifulsoup

于 2024-06-11 21:21:16 首次发布

本文链接：https://blog.youkuaiyun.com/m0_72904009/article/details/139609015

版权

爬取房源的网站是链家

爬取房源代码scrap.py

import requests
from bs4 import BeautifulSoup
import csv

# 网址
url = "https://bj.lianjia.com/zufang/pg{page}#contentList"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
}

# 初始化页码
page = 1

# 打开文件，准备写入
with open('house.csv', 'w', newline='', encoding='utf-8') as file:
    writer = csv.writer(file)
    # 写入表头
    #writer.writerow(["名字", "地址", "价格（元/月）", "链接"])

    while page <= 10:
        # 抓取目标页面
        response = requests.get(url.format(page=page), headers=headers)

        # 创建一个BeautifulSoup对象，获取页面正文
        soup = BeautifulSoup(response.text, 'html.parser')
        house = soup.find_all('a', class_='content__list--item--aside')
        money = soup.find_all('span', class_='content__list--item-price')
        locations = soup.find_all('p', class_='content__list--item--des')

        for i in range(len(house)):
            title = house[i].get('title')
            href = "https://bj.lianjia.com"+house[i].get('href')
            price = money[i].text
            location = locations[i]
            tags_a = location.find_all('a')
            location_texts = [tag_a.text for tag_a in tags_a]
            location_str = ''.join(location_texts)
            if(title == None or href == None or price == None or tags_a == None):
                continue
            # 写入一行数据
            writer.writerow([title,location_str, price, href])

        page += 1

会在同目录下生成一个house.csv

这个是index.html页面

# 地图程序 位于index.py文件 第104行
<html>
    <head>
        <meta charset="utf-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge">
        <meta name="viewport" content="initial-scale=1.0, user-scalable=no, width=device-width">
        <title>高德地图找房</title>
        <link rel="stylesheet" href="http://cache.amap.com/lbs/static/main1119.css" />
        <link rel="stylesheet" href="http://cache.amap.com/lbs/static/jquery.range.css" />
        <script src="http://cache.amap.com/lbs/static/jquery-1.9.1.js"></script>
        <script src="http://cache.amap.com/lbs/static/es5.min.js"></script>
       <script type="text/javascript">
        window._AMapSecurityConfig = {
            securityJsCode:'*********************************',
            
        }
</script>
<script type="text/javascript" src="https://webapi.amap.com/maps?v=1.4.15&key=*******************&plugin=AMap.ArrivalRange,AMap.Scale,AMap.Geocoder,AMap.Transfer,AMap.Autocomplete"></script>
        <script src="http://cache.amap.com/lbs/static/jquery.range.js"></script>
    <style>
      /*面板控制样式*/
      .control-panel {
        position: absolute;
        top: 30px;
        right: 20px;
      }
      /*面板内容样式*/
      .control-entry {
        width: 280px;
        background-color: rgba(119, 136, 153, 0.8);
        font-family: fantasy, sans-serif;
        text-align: left;
        color: white;
        overflow: auto;
        padding: 10px;
        margin-bottom: 10px;
      }
      /*文字与右侧的距离*/
      .control-input {
        margin-left: 120px;
      }
      /*输入框宽度*/
      .control-input input[type='text'] {
        width: 160px;
      }
      /*文字样式*/
      .control-panel label {
        float: left;
        width: 120px;
      }
      /*路线规划信息窗体样式*/
      #transfer-panel {
        position: absolute;
        background-color: white;
        max-height: 80%;
        overflow-y: auto;
        top: 30px;
        left: 20px;
        width: 250px;
      }
    </style>
  </head>

  <body>
    <div id="container"></div>
    <!--显示输入地址面板-->
    <div class="control-panel">
      <div class="control-entry">
        <label>选择工作地点：</label>
        <div class="control-input">
          <input id="work-location" type="text" />
        </div>
      </div>
      <!--显示选择交通的面板-->
      <div class="control-entry">
        <label>选择通勤方式：</label>
        <div class="control-input">
          <input
            type="radio"
            name="vehicle"
            value="SUBWAY,BUS"
            onClick="takeBus(this)"
            checked
          />
          公交+地铁
          <input
            type="radio"
            name="vehicle"
            value="SUBWAY"
            onClick="takeSubway(this)"
          />
          地铁
        </div>
      </div>
      <!--显示导入房源的面板-->
      <div class="control-entry">
        <label>导入房源文件：</label>
        <div class="control-input">
          <input type="file" name="file" onChange="importRentInfo(this)" />
        </div>
      </div>
    </div>
    <div id="transfer-panel"></div>
    <script>
     var map = new AMap.Map('container', {
  resizeEnable: true,
  zoomEnable: true,
  center: [116.397428, 39.90923],
  zoom: 11,

});
//添加标尺
var scale = new AMap.Scale();
map.addControl(scale);
//经度、纬度、时间、通勤方式（默认是地铁+公交）
var arrivalRange = new AMap.ArrivalRange();
var x,
  y,
  t,
  vehicle = 'SUBWAY,BUS';
//工作地点，工作标记
var workAddress, workMarker;
//房源标记数组
var rentMarkerArray = [];
//多边形数组，存储到达范围的计算结果
var polygonArray = [];
//路线规划
var amapTransfer;
//信息窗体对象
var infoWindow = new AMap.InfoWindow({
  offset: new AMap.Pixel(0, -30),
});
//地址自动补全对象
var auto = new AMap.Autocomplete({
  input: 'work-location',
});
//添加事件监听，在选择完地址后调用workLocationSelected
AMap.event.addListener(auto, 'select', workLocationSelected);

function takeBus(radio) {
  vehicle = radio.value;
  loadWorkLocation();
}

function takeSubway(radio) {
  vehicle = radio.value;
  loadWorkLocation();
}
//导入房源信息触发的方法
function importRentInfo(fileInfo) {
  //获取房源文件名称
  var file = fileInfo.files[0].name;
  loadRentLocationByFile(file);
}
//选择工作地点后触发的方法
function workLocationSelected(e) {
  //更新工作地点，加载到达范围
  workAddress = e.poi.name;
  //调用加载1小时到达区域的方法
  loadWorkLocation();
}
//加载工作地点标记
function loadWorkMarker(x, y, locationName) {
  workMarker = new AMap.Marker({
    map: map,
    title: locationName,
    icon: 'http://webapi.amap.com/theme/v1.3/markers/n/mark_r.png',
    position: [x, y],
  });
}
//加载到达范围
function loadWorkRange(x, y, t, color, v) {
  arrivalRange.search(
    [x, y],
    t,
    function (status, result) {
      if (result.bounds) {
        for (var i = 0; i < result.bounds.length; i++) {
          //多边形对象
          var polygon = new AMap.Polygon({
            map: map,
            fillColor: color, //填充色
            fillOpacity: '0.4', //透明度
            strokeColor: color,
            strokeOpacity: '0.8',
            strokeWeight: 1, //线宽
          });
          //到达范围的多边形路径
          polygon.setPath(result.bounds[i]);
          //增加多边形
          polygonArray.push(polygon);
        }
      }
    },
    {
      policy: v,
    }
  );
}
//添加房源标记
function addMarkerByAddress(address) {
  //地理编码对象
  var geocoder = new AMap.Geocoder({
    city: '北京',
    radius: 1000,
  });
  //获取位置
  geocoder.getLocation(address, function (status, result) {
    if (status === 'complete' && result.info === 'OK') {
      //获取地理编码
      var geocode = result.geocodes[0];
      //标记对象
      rentMarker = new AMap.Marker({
        map: map, //显示标记的地图
        title: address, //鼠标移动至标记时所显示的文字
        //标记图标地址
        icon: 'http://webapi.amap.com/theme/v1.3/markers/n/mark_b.png',
        //位置
        position: [geocode.location.getLng(), geocode.location.getLat()],
      });
      rentMarkerArray.push(rentMarker);
      //相关房源网络地址
      rentMarker.content =
        "<div>房源：<a target = '_blank' href='https://bj.58.com/pinpaigongyu/?key=" +
        address +
        "'>" +
        address +
        '</a><div>';
      //标记的事件处理
      rentMarker.on('click', function (e) {
        //设置信息窗体显示的内容
        infoWindow.setContent(e.target.content);
        infoWindow.open(map, e.target.getPosition());
        //路线规划是否清除
        if (amapTransfer) amapTransfer.clear();
        //换乘对象
        amapTransfer = new AMap.Transfer({
          map: map,
          policy: AMap.TransferPolicy.LEAST_TIME,
          city: '北京市',
          panel: 'transfer-panel',
        });
        //根据起、终点坐标查询换乘路线
        amapTransfer.search(
          [
            {
              keyword: workAddress,
            },
            {
              keyword: address,
            },
          ],
          function (status, result) {}
        );
      });
    }
  });
}
//清除已有的到达区域
function delWorkLocation() {
  if (polygonArray) map.remove(polygonArray);
  if (workMarker) map.remove(workMarker);
  polygonArray = [];
}
//清除现有的房源标记
function delRentLocation() {
  if (rentMarkerArray) map.remove(rentMarkerArray);
  rentMarkerArray = [];
}
//加载1小时到达区域
function loadWorkLocation() {
  //清除已有的到达区域
  delWorkLocation();
  //创建地址坐标对象
  var geocoder = new AMap.Geocoder({
    city: '北京',
    radius: 1000,
  });
  //获取位置
  geocoder.getLocation(workAddress, function (status, result) {
    if (status === 'complete' && result.info === 'OK') {
      var geocode = result.geocodes[0]; //获取地址编码
      x = geocode.location.getLng(); //经度
      y = geocode.location.getLat(); //纬度
      //加载工作地点标记
      loadWorkMarker(x, y);
      //加载工作地点1小时内到达的范围
      loadWorkRange(x, y, 60, '#3f67a5', vehicle);
      //地图移动到工作地点的位置
      map.setZoomAndCenter(12, [x, y]);
    }
  });
}
//加载房源位置
function loadRentLocationByFile(fileName) {
  //清除现有的房源标记
  delRentLocation();
  //所有的地点都记录在集合中
  var rent_locations = new Set();
  //获取文件中的房源信息
  $.get(fileName, function (data) {
    //分割信息
    data = data.split('\n');
    //遍历房源位置
    data.forEach(function (item, index) {
      rent_locations.add(item.split(',')[1]);
    });
    rent_locations.forEach(function (element, index) {
      //加上房源标记
      addMarkerByAddress(element);
    });
  });
}
    </script>
  </body>
</html>

其中的密钥自己去高德开放API申请，此处用********代替