Python爬虫爬取数据存入MongoDB

最新推荐文章于 2023-05-25 18:09:36 发布

原创最新推荐文章于 2023-05-25 18:09:36 发布 · 3.8k 阅读

16 ·

CC 4.0 BY-SA版权

文章标签：

#网络爬虫 #爬虫存入MongoDB

Python爬虫同时被 2 个专栏收录

7 篇文章

订阅专栏

Mongodb

2 篇文章

订阅专栏

本文介绍了一个使用Python实现的针对58同城网站的信息抓取项目。通过发送HTTP请求获取页面内容，并利用BeautifulSoup进行解析，提取商品标题、价格、发布时间和区域等信息，最终将数据存储到MongoDB数据库中。

from bs4 import BeautifulSoup
import requests
import time
import pymongo

client = pymongo.MongoClient('Localhost', 27017)
ceshi = client['ceshi']
url_list = ceshi['url_list3']
item_info = ceshi['item_info3']
def get_links_form(channel, pages, who_sells=0):
    
    list_view = '{}{}/pn{}/'.format(channel, str(who_sells), str(pages))
    wb_data = requests.get(list_view)
    time.sleep(1)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    if soup.find('td' 't'):
        
        for link in soup.select('td.t a.t'):
            item_link = link.get('href').split('?')[0]
            url_list.insert_one({'url': item_link})
            print(item_link)
    else:
        pass
#get_links_form('http://bj.58.com/shuma/',2)  
def get_item_info(url):
    wb_data = requests.get(url)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    no_longer_exit = '404' in soup.find('script', type = "text/javascript").get('src').split('/')
    if no_longer_exit:
        pass
    else:
        
        title = soup.title.text
        price = soup.select('span.price.c_f50')[0].text
        date = soup.select('.time')[0].text
        area = list(soup.select('.c_25d a')[0].stripped_strings) if soup.find_all('span', 'c_25d') else None
        item_info.insert_one({'title':title, 'price':price, 'date':date , 'area':area})
        print({'title':title, 'price':price, 'date':date , 'area':area})
        
get_item_info("http://bj.58.com/diannao/31994026546616x.shtml")