Beautiful SOAP 爬网页

本文介绍了一个使用Python Beautiful Soup实现的网络爬虫实例,该爬虫用于抓取CVE漏洞信息页面,并解析关键数据如漏洞名称、严重程度等。

摘要生成于 C知道 ,由 DeepSeek-R1 满血版支持, 前往体验 >


Python Beautiful SOAP 是一款强大的html解析工具,堪称网络爬虫利器。

下面代码为工具cvelist.csv文件中的CVE ID, 分别爬出该CVE信息的一段代码。供记录。


# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import time
import os
import copy
import random
from urllib2 import Request, urlopen, HTTPError
import logging
import json
from bs4 import BeautifulSoup 
import json
import codecs
import gevent
import logging


URL = "http://cve.scap.org.cn/%s.html"
def fetchCVE(sid):
    sid = "CVE-"+str(sid).strip()
    request_url = URL %(sid)
    request_settings = { 'content-type': 'text/plain','Accept-Encoding':'deflate','User-Agent':'User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'}
    req = Request(request_url,headers=request_settings)
    content = ""
    try:
        response = urlopen(req)
        content = response.read().decode('utf8')
    except HTTPError, e:
        pass
    except Exception,e:
        pass
    return content

def fetchCVEByList(sidList,sidContentPair):
    length = len(sidList)
    count=0
    for sid in sidList:
        count = count+1
        debugstr = "Process %d of %d\n" % (count,length)
        content = fetchCVE(sid)
        sidContentPair[sid.strip()] = content
        fp = open("./file/"+sid.strip(), "w")
        fp.write(content.encode("utf-8"))
        fp.close()

def parseTD(table) :
    soup = BeautifulSoup(table,"lxml")
    tds=soup.find_all("td")
    return tds

def getScoreAndSeverity(table):
    tds = parseTD(table)
    if len(tds)>2:
        return (tds[2].string,tds[1].string)
    else:
        return ("","")
def getPlatform(table):
    tds = parseTD(table)
    content  =""
    for td in tds:
        if td.string!=None:
            content = content + "\n"+td.string
    return content

def getSummary(summary):
    soup = BeautifulSoup(summary,"lxml")
    strongs=soup.find_all("strong")
    return strongs[0].string

def writeCVEList(sidContentPair):
    length = 4152
    counter =0;
    logging.info("begin")
    sidInfoDic= {}
    for sid in sidContentPair.keys():
        debugstr = "process %d of total %d rule: SUCCEED\n"
        counter = counter+1
        content = sidContentPair.get(sid)
        if (content== ""):
            logging.error("sid:"+sid+ " content is none")
            continue
        try:
            soup = BeautifulSoup(content,"lxml")
            summary=soup.find_all("div", {'class':'summary'})
            cvsstable=soup.find_all(id="cvss")
            cpetable=soup.find_all(id="cpe")
            (severity,score) = getScoreAndSeverity(cvsstable[0].encode("utf-8"))
            if(severity=="" or score==""):
                logging.error("sid %s no score", sid)
            name = ""
            for content in summary[0].contents:
                if(content.encode("utf-8").find("strong")!=-1):
                    name = getSummary(content.encode("utf-8"))
            platform = getPlatform(cpetable[0].encode("utf-8"))
            sidInfoDic[sid]=[name,score.strip(),severity,platform]
            logging.info(debugstr , counter,length)
        except Exception,e:
            debugstr = "process %d of total %d rule: FAIL,sid="+sid+"\n"
            logging.exception(e)
            logging.info(debugstr , counter,length)
    #wstr = json.dumps(sidInfoDic, ensure_ascii=False)
    fp = open("result.json", "w")
    json.dump( sidInfoDic,fp, ensure_ascii=False,indent=4)
    fp.close()

def dumpResult():
    sidContentPair = {}
    fp = open("cvelist.csv",'r')
    lines = fp.readlines()
    fp.close()
    length = len(lines)
    threadNumber = length/500+1
    taskPerThread = 500
    threadList = []
    for i in xrange(threadNumber+1):
        taskBegin = i* taskPerThread
        taskEnd = (i+1)* taskPerThread
        if(taskEnd>length):
            taskEnd = length
        t = gevent.spawn(fetchCVEByList, lines[taskBegin:taskEnd],sidContentPair)
        threadList.append(t)
    gevent.joinall(threadList)
    writeCVEList(sidContentPair)

def dumpResultByFile():
    sidContentPair = {}
    #cve 文件,一行一个cve id
    fp = open("cvelist.csv",'r')
    lines = fp.readlines()
    fp.close()
    for line in lines:
        fp = open("./file/"+line.strip(), "r")
        content = fp.read()
        fp.close()
        sidContentPair[line.strip()] = content
    writeCVEList(sidContentPair)
    
if __name__=='__main__':
    #dumpResult()
    dumpResultByFile()


Python中,BeautifulSoup是一个强大的库,用于从HTML和XML文档中提取数据,而并非直接用于抓取图片。若你想用它抓取网页中的图片,通常需要结合其他库如requests来获取网页内容,然后使用BeautifulSoup解析页面结构。 以下是一个基本步骤: 1. **安装依赖库**: 首先确保已安装`requests`和`beautifulsoup4`库,如果没有,可以使用pip安装: ``` pip install requests beautifulsoup4 ``` 2. **发送HTTP请求获取网页**: 使用requests库发送GET请求获取网页源代码: ```python import requests url = "http://example.com" # 替换为你想抓取的网页URL response = requests.get(url) html_content = response.text ``` 3. **解析HTML内容**: 解析响应内容,找到所有的图片标签 `<img>`: ```python from bs4 import BeautifulSoup soup = BeautifulSoup(html_content, 'html.parser') img_tags = soup.find_all('img') # 获取所有img元素 ``` 4. **提取图片链接**: 从每个img标签中提取出src属性(图片链接): ```python image_links = [img['src'] for img in img_tags] ``` 5. **下载图片**: 可以用`os`模块创建目录并使用`urllib.request`下载图片,注意处理可能出现的网络错误: ```python import os if not os.path.exists("images"): os.makedirs("images") for link in image_links: try: response = requests.get(link, stream=True) filename = os.path.join("images", os.path.basename(link)) with open(filename, 'wb') as f: f.write(response.content) except Exception as e: print(f"Error downloading {link}: {e}") ```
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值