为自己的爬虫快速增加headers

原生请求头字符串

Python

raw_headers = """Host: open.tool.hexun.com Pragma: no-cache Cache-Control: no-cache User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36 Accept: */* Referer: http://stock.hexun.com/gsxw/ Accept-Encoding: gzip, deflate Accept-Language: zh-CN,zh;q=0.9,en;q=0.8"""

1 2 3 4 5 6 7 8 9

raw_headers = """Host: open.tool.hexun.com Pragma: no-cache Cache-Control: no-cache User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36 Accept: */* Referer: http://stock.hexun.com/gsxw/ Accept-Encoding: gzip, deflate Accept-Language: zh-CN,zh;q=0.9,en;q=0.8"""

将字符串转为字典

Python

split(sep=None, maxsplit=None) headers = dict([line.split(": ",1) for line in raw_headers.split("\n")]) 输出测试 print(headers) """ { 'Accept-Language': ' zh-CN,zh;q=0.9,en;q=0.8', 'Accept-Encoding': ' gzip, deflate', 'Accept': ' */*', 'User-Agent': ' Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36', 'Host': ' open.tool.hexun.com', 'Referer': ' http://stock.hexun.com/gsxw/', 'Pragma': ' no-cache', 'Cache-Control': ' no-cache' } """

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19

split ( sep = None , maxsplit = None ) headers = dict ( [ line . split ( ": " , 1 ) for line in raw_headers . split ( "\n" ) ] )   输出测试 print ( headers ) """ {     'Accept-Language': ' zh-CN,zh;q=0.9,en;q=0.8',     'Accept-Encoding': ' gzip, deflate',     'Accept': ' */*',     'User-Agent': ' Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',     'Host': ' open.tool.hexun.com',     'Referer': ' http://stock.hexun.com/gsxw/',     'Pragma': ' no-cache',     'Cache-Control': ' no-cache' }   """

封装为函数

Python

def get_headers(header_raw): """ 通过原生请求头获取请求头字典 :param header_raw: {str} 浏览器请求头 :return: {dict} headers """ return dict(line.split(": ", 1) for line in header_raw.split("\n")) def get_cookies(cookie_raw): """ 通过原生cookie获取cookie字段 :param cookie_raw: {str} 浏览器原始cookie :return: {dict} cookies """ return dict(line.split("=", 1) for line in cookie_raw.split("; "))

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17

def get_headers ( header_raw ) :      """     通过原生请求头获取请求头字典     :param header_raw: {str} 浏览器请求头     :return: {dict} headers     """      return dict ( line . split ( ": " , 1 ) for line in header_raw . split ( "\n" ) )     def get_cookies ( cookie_raw ) :      """     通过原生cookie获取cookie字段     :param cookie_raw: {str} 浏览器原始cookie     :return: {dict} cookies     """      return dict ( line . split ( "=" , 1 ) for line in cookie_raw . split ( "; " ) )

• zeropython 微信公众号 5868037 QQ号 5868037@qq.com QQ邮箱