python爬虫笔记（一）

最新推荐文章于 2024-03-01 23:47:59 发布

原创最新推荐文章于 2024-03-01 23:47:59 发布 · 289 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#python爬虫

python爬虫专栏收录该内容

5 篇文章

订阅专栏

本文详细介绍了爬虫系统的架构组成，包括调度器、URL管理器、网页下载器和网页解析器，并提供了使用urllib2、requests和BeautifulSoup进行网页下载和解析的实例代码。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1.架构

调度器：负责协调工作
URL管理器：通过内存、数据库、缓存数据库实现
网页下载器：urllib2(登录、代理、cookie)、requests
网页解析器：正则表达式、html.parser、beautifulSoup(一锅汤)、lxml(可以解析xml和html)
应用程序：

2.urllib2实现下载网页的三种方式

#!/usr/bin/python
# -*- coding: UTF-8 -*-
 
import cookielib
import urllib2
 
url = "http://www.baidu.com"
response1 = urllib2.urlopen(url)
print "第一种方法"
#获取状态码，200表示成功
print response1.getcode()
#获取网页内容的长度
print len(response1.read())
 
print "第二种方法"
request = urllib2.Request(url)
#模拟Mozilla浏览器进行爬虫
request.add_header("user-agent","Mozilla/5.0")
response2 = urllib2.urlopen(request)
print response2.getcode()
print len(response2.read())
 
print "第三种方法"
cookie = cookielib.CookieJar()
#加入urllib2处理cookie的能力
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
urllib2.install_opener(opener)
response3 = urllib2.urlopen(url)
print response3.getcode()
print len(response3.read())
print cookie

python3中导入urllib.request代替urllib2

3.使用request库下载网页（例子：下载小说）

1.抓取网页
import requests
 
if __name__ == '__main__':
    target = 'http://www.biqukan.com/1_1094/5403177.html'
    req = requests.get(url=target)
    print(req.text)

2.提取内容
from bs4 import BeautifulSoup
import requests
if __name__ == "__main__":
     target = 'http://www.biqukan.com/1_1094/5403177.html'
     req = requests.get(url = target)
     html = req.text
     bf = BeautifulSoup(html)
     texts = bf.find_all('div', class_ = 'showtxt')
     print(texts)

3.去除标签
from bs4 import BeautifulSoup
import requests
if __name__ == "__main__":
     target = 'http://www.biqukan.com/1_1094/5403177.html'
     req = requests.get(url = target) 
     html = req.text
     bf = BeautifulSoup(html)
     texts = bf.find_all('div', class_ = 'showtxt')
     print(texts[0].text.replace('\xa0'*8,'\n\n'))

*note：
find_all匹配的返回的结果是一个列表。
提取匹配结果后，使用text属性，提取文本内容，滤除br标签。
随后使用replace方法，剔除空格，替换为回车进行分段。
在html中是用来表示空格的。
replace(‘\xa0’8,’\n\n’)就是去掉下图的八个空格符号，并用回车代替：

 4.获取小说目录
from bs4 import BeautifulSoup
import requests
if __name__ == "__main__":
     target = 'http://www.biqukan.com/1_1094/'
     req = requests.get(url = target)
     html = req.text
     div_bf = BeautifulSoup(html)
     div = div_bf.find_all('div', class_ = 'listmain')
     print(div[0])


5.提取标题和链接
from bs4 import BeautifulSoup
import requests
if __name__ == "__main__":
     server = 'http://www.biqukan.com/'
     target = 'http://www.biqukan.com/1_1094/'
     req = requests.get(url = target) html = req.text
     div_bf = BeautifulSoup(html)
     div = div_bf.find_all('div', class_ = 'listmain')
     a_bf = BeautifulSoup(str(div[0]))
     a = a_bf.find_all('a')
     for each in a:
          print(each.string, server + each.get('href'))

4.使用urllib库下载网页（）

1.获取网页
import urllib.request #引入urllib库

response = urllib.request.urlopen("https://tieba.baidu.com/index.html")  #发出请求并且接收返回文本对象
html = response.read()   #调用read()进行读取
print(html)  #打印

2.将网页转换为文本：utf-8格式
import urllib.request #引入urllib库

response = urllib.request.urlopen("https://tieba.baidu.com/index.html")  #发出请求并且接收返回文本对象
html = response.read().decode('utf-8')   #调用read()进行读取，转换为utf-8的编码
print(html)  #打印

3.获取网页的编码
import urllib.request #引入urllib库
import chardet

response = urllib.request.urlopen("https://tieba.baidu.com/index.html")  #发出请求并且接收返回文本对象
html = response.read()   # 调用read()进行读取
chardit1 = chardet.detect(html) #获取文本编码
print(chardit1)

print(html.decode(chardit1['encoding']))


note:chardit1是一个json字符串
{'encoding': 'utf-8', 'confidence': 0.99, 'language': ''}

5.将爬取的内容存到txt中

import urllib.request
import re
data=urllib.request.urlopen("https://read.douban.com/provider/all").read()
data=data.decode("utf-8")
pat='="name">(.*?)<'   
mydata=re.compile(pat).findall(data)
print(mydata)
file=open("E:/fh.txt","w")  
for i in range (len(mydata)):
    file.write(mydata[i]+"\n")
file.close()

6.使用Beautiful Soup解析html文件

import re
 
from bs4 import BeautifulSoup
html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
#创建一个BeautifulSoup解析对象
soup = BeautifulSoup(html_doc,"html.parser",from_encoding="utf-8")
#获取所有的链接
links = soup.find_all('a')
print "所有的链接"
for link in links:
    print link.name,link['href'],link.get_text()
 
print "获取特定的URL地址"
link_node = soup.find('a',href="http://example.com/elsie")
print link_node.name,link_node['href'],link_node['class'],link_node.get_text()
 
print "正则表达式匹配"
link_node = soup.find('a',href=re.compile(r"ti"))
print link_node.name,link_node['href'],link_node['class'],link_node.get_text()
 
print "获取P段落的文字"
p_node = soup.find('p',class_='story')
print p_node.name,p_node['class'],p_node.get_text()