import requests
import urllib.request as ur
from bs4 import BeautifulSoup
url="https://blog.youkuaiyun.com/qq_41896729?spm=1001.2014.3001.5343"
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}
ss=None
try:
r=requests.get(url,headers=headers,timeout=30)
soup = BeautifulSoup(r.text, 'lxml')
# 下面的这种方法会报错 TypeError("'NoneType' object is not callable")
# ss=soup.findall("class",class_="article-item-box csdn-tracking-statistics")
# 找到所有class为article-item-box csdn-tracking-statistics的节点
for s in soup.findAll(name="div", attrs={"class" :"article-item-box csdn-tracking-statistics"}):
for ss in s.findAll(name="h4"):
sss=ss.find(name="a",href=True);
print("文章标题:"+ss.getText().replace("原","").strip()+"\n文章链接:"+sss['href'])
req = ur.Request(sss['href'])
req.add_header("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)")
file = ur.urlopen(req)
content = file.read()
# print(content)
except Exception as e:
print("出现异常------异常信息:"+repr(e));