python爬虫之Urllib

 

什么是Urllib

Python内置的HTTP请求库

urllib.request      请求模块

urllib.error          异常处理模块

urllib.parse         url解析模块

urllib.robotparser       robots.txt  解析模块

相⽐Python2变化

Python2 

import urllib2

response = urllib2.urlopen('http://www.baidu.com')

Python3

import urllib.request

response = urllib.request.urlopen('http://www.baidu.com')

#get请求获取网页

import urllib.request

response = urllib.request.urlopen('http://www.baidu.com')
html = response.read().decode('utf-8')
print(html)

 

#post请求获取网页

import urllib.parse
import urllib.request

data = bytes(urllib.parse.urlencode({'world':'hello'}),encoding='utf-8')
response = urllib.request.urlopen('http://httpbin.org/post',data)
print(response.read())

超时设置

import socket
import urllib.request

response = urllib.request.urlopen('http://httpbin.org/get',timeout=1)   #timeout=1表示超时1秒刷新

print(response.read())

 

在0.1秒内得到响应，不然抛出异常打印输出“time out”

import socket
import urllib.request
import urllib.error

try:
    response = urllib.request.urlopen('http://httpbin.org/get',timeout=0.1)  #在0.1秒内得到响应，不然打印输出“time out”
except urllib.error.URLError as e:
    if isinstance(e.reason,socket.timeout):
        print('TIME OUT')

响应

相应类型

import urllib.request

response = urllib.request.urlopen('https://www.python.org')
print(type(response))

 

状态码、响应头

import urllib.request

response = urllib.request.urlopen('https://www.python.org')
print(response.status)   #状态码
print(response.getheaders())  #响应头
print(response.getheader('Server'))  #服务器类型

状态码200表示成功

获取响应内容

import urllib.request

response = urllib.request.urlopen("https://www.python.org")
print(response.read().decode('utf-8')) #read()表示获取响应内容，也叫响应体内容，decode('utf-8')表示响应内容用utf-8编码

Request

request = urllib.request.Request("https://www.python.org")
response = urllib.request.urlopen(request)
print(response.read().decode('utf-8')) #read()表示获取响应内容，也叫响应体内容，decode('utf-8')表示响应内容用utf-8编码

 

from urllib import request,parse

url = "http://httpbin.org/post"

headers = {' User-Agent':"Mozilla/4.0(compatible; MSIE 5.5; Windows NT),"
                          "Host:'httpbin.org'"}

dict = {
    'name':'Germey'
}

data = bytes(parse.urlencode(dict),encoding='utf8')
req = request.Request(url=url,data=data,headers=headers,method='POST')
response = request.urlopen(req)
print(response.read().decode('utf-8'))    #可能运行出错，是http://httpbin.org/post的问题，试着浏览器访问

 

 

添加一个add_header方法

 

from urllib import request,parse

url = "http://httpbin.org/post"

dict = {
    'name':'Germey'
}
data = bytes(parse.urlencode(dict),encoding='utf8')
req = request.Request(url=url,data=data,method='POST')
req.add_header('User-Agent',"Mozilla/4.0(compatible; MSIE 5.5; Windows NT)")
response = request.urlopen(req)
print(response.read().decode('utf-8'))

 

 

伪装ip