模拟浏览器对百度进行爬虫

import urllib.request
import random

#url
url="http://www.baidu.com"

'''
#模拟单个请求头
headers={
    "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
}
#模拟请求体
req=urllib.request.Request(url,headers=headers)
'''

#模拟多个浏览器的请求头，这里以谷歌和火狐为例
AgentList=[
"Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/63.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
]
a=random.choice(AgentList) #随机选择一个请求头
#模拟请求体
req=urllib.request.Request(url)
#添加请求头
req.add_header("User-Agent",a)

response=urllib.request.urlopen(req)

data=response.read().decode("utf-8")
print(data)

转载于:https://www.cnblogs.com/wfw001-2018/p/10030597.html