asyncio/aiohttp 异步爬取douban-优快云博客

本文链接：https://blog.youkuaiyun.com/shuishou07/article/details/78488968

本文介绍了一个使用Python异步I/O库aiohttp和BeautifulSoup实现的简单爬虫程序，该程序能够从豆瓣网站抓取小说标签页的前20页数据，并解析每一页中的小说名称。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import aiohttp
import asyncio
import async_timeout
from bs4 import BeautifulSoup


#信号量，控制协程数，防止爬的过快
sema = asyncio.Semaphore(3)

#解析html,获取小说书名
async def parse_html(text):
    soup = BeautifulSoup(text, 'lxml')
    items = soup.select('.subject-list .subject-item .info')
    titles = [item.find('a')['title'] for item in items]
    print(titles)

#获取html页面
async def get_html(url):
    async with aiohttp.ClientSession() as sess:
        with async_timeout.timeout(10):#设置请求的最长时间为10s
            async with sess.get(url, proxy="http://54.222.232.0:3128") as res:
                text = await res.text()
                return text


async def crawl_douban(url):
    with(await sema):
        text = await get_html(url)
        await parse_html(text)

def crawl():
    # 豆瓣小说首页
    start_url = 'https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4'
    tasks = [crawl_douban(start_url)]
    # 第2到20页的url，加入到tasks中
    for i in range(1, 20):
        url = '{}?start={}&type=T'.format(start_url, i * 20)
        tasks.append(crawl_douban(url))

    loop = asyncio.get_event_loop()
    #loop.run_until_complete(asyncio.gather(*tasks))
    loop.run_until_complete(asyncio.wait(tasks))
    loop.close()

if __name__ == '__main__':
    crawl()