day2-requests模块基础

requests模块

• python中原生的一款基于网络请求的模块，功能非常强大，简单便捷，效率极高
• 作用：模拟浏览器发请求。
• 如何使用：( requests模块的编码流程)
  • 指定urL
  • 发起请求
  • 获取响应数据
  • 持久化存储
• 环境安装：
  pip install requests

实战编码：

1. 需求：爬取搜狗首页的页面数据

import requests


def get_sogou():
    #step1 指定URL
    url = 'https://www.sogou.com/'
    #step2 发起请求
    response = requests.get(url=url)
    #step3 获取响应数据，txt返回的是字符串形式的响应数据
    page_txt = response.text
    print(page_txt)
    #step4 持久化存储
    with open('./sogou.html','w',encoding='utf-8') as f:
        f.write(page_txt)
    print('爬虫结束-----')



if  __name__ == '__main__':
    get_sogou()

实战巩固

1. 需求：爬取搜狗指定词条对应的搜索结果页面（简易网页采集器）

import requests


#UA：User-Agent(请求载体的身份标识)
#UA检测：门户网站的服务器会检测对应请求的载体身份标识，如果检测到请求的载体身份标识为某一款浏览器，说明
#该请求是一个正常的请求，但是，如果检测请求的载体身份标识不是基于某一款浏览器，则表示该请求为不正常的请求
#则服务器就很可能会拒绝该请求
#UA伪装：让爬虫对应的请求载体身份标识伪装成某一款浏览器

def  get_sogou():
    #UA伪装，将对应的User-Agent封装到一个字典中
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36'
    }
    url = 'https://www.sogou.com/web'
    #处理url携带的参数：封装到字典中
    kw = input('请输入要查询的字符：')
    param = {
        'query':kw
    }
    response = requests.get(url=url,params=param,headers=headers)
    page_txt = response.text
    file_name = kw + '.html'
    with open(file_name,'w',encoding='utf-8') as f:
        f.write(page_txt)
    print('保存成功')


if __name__ == '__main__':
    get_sogou()

2. 需求：破解百度翻译

import requests
import json

def get_baidu():
    url = 'https://fanyi.baidu.com/sug'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36'
    }
    kw = input('请输入要翻译的字符')
    data = {
        'kw':kw
    }
    response = requests.post(url=url,data=data,headers=headers)
    dic_obj = response.json()
    file = kw+'.json'
    f =  open(file,'w',encoding='utf-8')
    json.dump(dic_obj,fp=f,ensure_ascii=False)
    print('翻译成功')


if __name__ == '__main__':
    get_baidu()

3. 需求：爬取豆瓣电影分类排行榜htps://movie.douban.com/中的电影详情数据

import json
import requests


def  get_douban():
    url = 'https://movie.douban.com/j/chart/top_list'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36'
    }
    param = {
        'type': '11',
        'interval_id': '100:90',
        'action':'',
        'start': '0',
        'limit': '20'
    }

    response = requests.get(url=url,params=param,headers=headers)
    list_data = response.json()
    f = open('./douban.json','w',encoding='utf-8')
    json.dump(list_data,fp=f,ensure_ascii=False)
    print('爬取成功')


if __name__ == '__main__':
    get_douban()

4. 需求：爬取肯德基餐厅查询http://www.kfc.comcn/kfccda/index.aspx中指定地点的餐厅数

import json

import requests


def get_kfc():
    url = 'http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36'
    }
    data = {
        'cname':'',
        'pid':'',
        'keyword': '深圳',
        'pageIndex': '1',
        'pageSize': '10'
    }
    params = {
        'op':'keyword'
    }
    response = requests.post(url=url,headers=headers,data=data,params=params)
    result = response.json()
    total = result['Table'][0]['rowcount']
    if total > 10:
        data['pageSize'] = total
        response = requests.post(url=url, headers=headers, data=data, params=params)
    result = response.json()
    f = open('./kfc.json','w',encoding='utf-8')
    json.dump(result,fp=f,ensure_ascii=False)
    print('爬取成功')


if __name__ == '__main__':
    get_kfc()