基础爬虫代码记录

spiderman.py  主服务
# -*- encoding=UTF-8 -*-
'''
author:vfast
name:spiderman
data:2021/6/28
'''

from dataoutput import DataOutput
from downloadhtml import HtmlDownload
from exparehtml import Exparsehtml
from urlmanager import UrlManager


class SpiderMan(object):
    def __init__(self):
        self.manager = UrlManager()
        self.downloader = HtmlDownload()
        self.parser = Exparsehtml()
        self.output = DataOutput()

    def crawl(self, root_url):
        # 添加入口url
        self.manager.add_new_url(root_url)
        # 判断url管理器中是否有新的url，同时判断抓取了多少个url
        while (self.manager.has_new_url() and self.manager.old_url_size() < 100):
            try:
                # 从url管理器获取新的url
                new_url = self.manager.get_new_url()
                # html下载器下载网页
                html = self.downloader.download(new_url)
                # HTML解析器抽取网页数据
                new_urls, data = self.parser.parser(new_url, html)
                # 将抽取的url添加到url管理器中
                self.manager.add_new_urls(new_urls)
                # 数据存储器存储文件
                self.output.store_data(data)
                print "已经抓取%s个链接" % self.manager.old_url_size()
            except Exception, e:
                print "crawl failed"
            # 数据存储器将文件输出成指定格式
        self.output.output_html()


if __name__ == '__main__':
    spider_man = SpiderMan()
    spider_man.crawl("http://baike.baidu.com/view/284853.htm")

urlmanager.py url管理服务

# -*- encoding=UTF-8 -*-
'''
author:vfast
name:spider
data:2021/6/24
'''


class UrlManager(object):
    def __init__(self):
        self.new_urls = set()  # 没有爬取的url集合
        self.old_urls = set()  # 已经爬取的url集合

    def has_new_url(self):
        '''
        判断是否有没有爬取的url
        :return:
        '''
        return self.new_url_size() != 0

    def get_new_url(self):
        '''
        获取一个未爬取的url
        :return:
        '''
        new_url = self.new_urls.pop()
        self.old_urls.add(new_url)
        return new_url

    def add_new_url(self, url):
        '''
        添加新的url到未爬取的url集合中
        :param url: url地址
        :return:
        '''
        if url is None:
            return
        if url not in self.new_urls and url not in self.old_urls:
            self.new_urls.add(url)

    def add_new_urls(self, urls):
        '''
        添加新的url到未爬取的url集合中
        :param urls: url集合
        :return:
        '''
        if urls is None or len(urls) == 0:
            return
        for url in urls:
            self.add_new_url(url)

    def new_url_size(self):
        '''
        获取未爬取的url集合大小
        :return:
        '''
        return len(self.new_urls)

    def old_url_size(self):
        '''
        获取已经爬取的url集合大小
        :return:
        '''
        return len(self.old_urls)

downloadhtml.py 下载html网页内容

# -*- encoding=UTF-8 -*-
'''
author:vfast
name:downloadhtml
data:2021/6/25
'''

import requests


class HtmlDownload(object):
    def download(self, url):
        if url is None:
            return None
        user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0'
        headers = {'User-Agent': user_agent}
        r = requests.get(url, headers=headers)
        if r.status_code == 200:
            r.encoding = 'utf-8'
            return r.text

        return None

exparehtml.py 解析HTML网页

# -*- encoding=UTF-8 -*-
'''
author:vfast
name:exparehtml
data:2021/6/25
'''

import re
import urlparse
from bs4 import BeautifulSoup


class Exparsehtml(object):
    def parser(self, page_url, html_cont):
        '''
        用于解析网页内容，抽取url和数据
        :param page_url: 下载页面的url
        :param html_cont: 下载的网页内容
        :return: 返回url和数据
        '''
        if page_url is None or html_cont is None:
            return
        soup = BeautifulSoup(html_cont, 'html.parser')
        new_urls = self._get_new_urls(page_url, soup)
        new_data = self._get_new_data(page_url, soup)
        return new_urls, new_data

    def _get_new_urls(self, page_url, soup):
        '''
        抽取新的url集合
        :param page_url: 下载页面的url
        :param soup:
        :return: 返回新的集合
        '''
        new_urls = set()
        # 抽取符合要求的a标签
        links = soup.find_all('a', href=re.compile(r'/view/\d+\.htm'))
        for link in links:
            new_url = link['href']
            new_full_url = urlparse.urljoin(page_url, new_url)
            new_urls.add(new_full_url)
        return new_urls

    def _get_new_data(self, page_url, soup):
        '''
        抽取有效的数据
        :param page_url: 下载页面的url
        :param soup:
        :return:  返回有效数据
        '''
        data = {}
        data['url'] = page_url
        title = soup.find('dd', class_='lemmaWgt-lemmaTitle-title').find('h1')
        data['title'] = title.get_text()
        summary = soup.find('div', class_='lemma-summary')

        data['summary'] = summary.get_text()
        return data

dataoutput.py  保存数据到文件

# -*- encoding=UTF-8 -*-
'''
author:vfast
name:dataoutput
data:2021/6/28
'''

import codecs


class DataOutput(object):
    def __init__(self):
        self.datas = []

    def store_data(self, data):
        if data is None:
            return
        self.datas.append(data)

    def output_html(self):
        fout = codecs.open('baike.html', 'w', encoding='utf-8')
        fout.write("<html>")
        fout.write("<body>")
        fout.write("<table>")
        for data in self.datas:
            fout.write("<tr>")
            fout.write("<td>%s</td>" % data['url'])
            fout.write("<td>%s</td>" % data['title'])
            fout.write("<td>%s</td>" % data['summary'])
            fout.write("</tr>")
            self.datas.remove(data)
        fout.write("</table>")
        fout.write("</body>")
        fout.write("</html>")
        fout.close()

来源：Python爬虫开发与项目实战,没有地址记录一下