从一个指定的网站（在这个例子中是 http://www.xvipxs.net/）抓取网页内容-优快云博客

from bs4 import BeautifulSoup
import urllib3,re,time

urllib3.disable_warnings()
domain='http://www.xvipxs.net/'
ad=re.compile('http.*')

class spide:
    def __init__(self) -> None:
        self.http = urllib3.PoolManager(cert_reqs='CERT_NONE')
        self.headers={
            'User-Agent':"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
            "Referer":domain,
            "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
            "Accept-Language":"zh-CN,zh;q=0.9"
        }

    def getPage(self,url):
        r = self.http.request('GET', domain+url,headers=self.headers)
        data=r.data.decode()
        soup = BeautifulSoup(data,'html.parser')
        # soup = BeautifulSoup(open('t.html'))
        title=soup.title.string
        title=title[title.find(' '):title.find('_')]
        print(url,title,int(time.time()))
        nextUrl=''
        botton=soup.find('div',class_="bottem2")
        botton=botton.find_all('a')
        for i in botton:
            if i.string=='下一章':
                nextUrl=i.attrs['href']
                break

        content=soup.find(id="content").text
        content=ad.sub('',content)
        if len(content)<100:
            print('字数过少，跳过')
            return (nextUrl,'',title)
        else:
            return (nextUrl,content,title)