python内置urllib库的使用

urllib库是python内置的一个http请求库

urllib.request 请求模块
urllib.error 异常处理模块
urllib.parse 解析模块

---

用法讲解
（1）简单的一个get请求

import urllib.request
reponse = urllib.request.urlopen('http://www.baidu.com')
print(reponse.read().decode('utf-8'))

（2）简单的一个post请求

import urllib.parse
import urllib.request
data = bytes(urllib.parse.urlencode({'hello':'world'}),encoding='utf-8')
reponse = urllib.request.urlopen('http://httpbin.org/post',data=data)
print(reponse.read())

（3）超时处理

import urllib.request
response = urllib.request.urlopen('http://httpbin.org/get',timeout=1)
print(response.read())

import urllib.request
import socket
import urllib.error
try:
    response = urllib.request.urlopen('http://httpbin.org/get',timeout=0.01)
except urllib.error.URLError as e:
    if isinstance(e.reason,socket.timeout):#判断错误原因
        print('time out!')

（4）打印出响应类型，状态码，响应头

import urllib.request
response=urllib.request.urlopen('http://www.baidu.com')
print(type(response))

import urllib.request
response = urllib.request.urlopen('http://www.baidu.com')
print(response.status)                  # 状态码  判断请求是否成功
print(response.getheaders())            # 响应头 得到的一个元组组成的列表
print(response.getheader('Server'))     #得到特定的响应头
print(response.read().decode('utf-8'))  #获取响应体的内容，字节流的数据，需要转成utf-8格式

 

（5）由于使用urlopen无法传入参数，需要声明一个request对象，通过这个对象来添加参数

import urllib.request
request = urllib.request.Request('https://python.org')   #由于urlopen无法传参数，声明一个Request对象
response = urllib.request.urlopen(request)
print(response.read().decode('utf-8'))

（6）分别创建字符串、字典等等来带入到request对象里面

from urllib import request,parse
url='http://httpbin.org/post'
headers={
    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
    'Host':'httpbin.org'
}
dict={
    'name':'jay'
}
data = bytes(parse.urlencode(dict),encoding='utf-8')
req=request.Request(url=url,data=data,headers=headers,method='POST')
response=request.urlopen(req)
print(response.read().decode('utf-8'))

（7）通过addheaders方法不断的向原始的requests对象里不断添加

from urllib import request,parse
url ='http://httpbin.org/post'
dict = {
    'name':'cq'
}
data=bytes(parse.urlencode(dict),encoding='utf-8')
req = request.Request(url=url,data=data,method='POST')
req.add_header('user-agent', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36')
response=request.urlopen(req)
print(response.read().decode('utf-8')

（8）打印出信息cookies
下面这段程序获取response后声明的cookie对象会被自动赋值

import http.cookiejar,urllib.request
cookie = http.cookiejar.CookieJar()
handerler=urllib.request.HTTPCookieProcessor(cookie)
opener=urllib.request.build_opener(handerler)
response=opener.open('http://www.baidu.com')  #获取response后cookie会被自动赋值
for item in cookie:
    print(item.name+'='+item.value)

（9）保存cookie文件,两种格式

import http.cookiejar,urllib.request
filename='cookie.txt'
cookie = http.cookiejar.MozillaCookieJar(filename)
handerler=urllib.request.HTTPCookieProcessor(cookie)
opener=urllib.request.build_opener(handerler)
response=opener.open('http://www.baidu.com')  #获取response后cookie会被自动赋值
cookie.save(ignore_discard=True,ignore_expires=True)  #保存cookie.txt文件

import http.cookiejar,urllib.request
filename='cookie2.txt'
cookie = http.cookiejar.LWPCookieJar(filename)
handerler=urllib.request.HTTPCookieProcessor(cookie)
opener=urllib.request.build_opener(handerler)
response=opener.open('http://www.baidu.com')  #获取response后cookie会被自动赋值
cookie.save(ignore_discard=True,ignore_expires=True)  #保存cookie.txt文件

（10）解析,将一个url解析

from urllib.parse import  urlparse
result = urlparse('http://www.baidu.com/index.html;user?id=5#comment')
print(result)#协议内容、路径、参数
print(type(result))

from urllib.parse import  urlparse
result = urlparse('www.baidu.com/index.html;user?id=5#comment',scheme='https')
print(result)



from urllib.parse import  urlparse
result = urlparse('http://www.baidu.com/index.html;user?id=5#comment',scheme='https')
print(result)

from urllib.parse import  urlparse
result = urlparse('http://www.baidu.com/index.html;user?id=5#comment',allow_fragments=False)  #会被拼接
print(result)

from urllib.parse import  urlparse
result = urlparse('http://www.baidu.com/index.html#comment',allow_fragments=False)  #会被拼接到path没有query
print(result)

（11）url拼接

from urllib.parse import urlunparse
data=['http','www.baidu.com','index.html','user','a=6','comment']
print(urlunparse(data))

from urllib.parse import  urljoin
#拼接两个url
#截图，以后面的为基准，有留下，没有拼接

print(urljoin('http://www.baidu.com','HAA.HTML'))
print(urljoin('https://wwww.baidu.com','https://www.baidu.com/index.html;question=2'))

#字典方式直接转换成url参数
from urllib.parse import urlencode
params = {
    'name':'germey',
    'age':'122'
}
base_url='http://www.baidu.com?'
url=base_url+urlencode(params)
print(url)