requests模拟登陆

第一种方式,用户名密码登陆:

#!/usr/bin/python3
#-*- coding:utf-8 -*-

import requests
import html5lib
import re
from bs4 import BeautifulSoup

s = requests.Session()

url_login = 'http://accounts.douban.com/login'
url_contacts = 'https://www.douban.com/people/163532496/'

formdata = {
    'redir':'https://www.douban.com',
    'form_email': 'yourUsername',
    'form_password': 'yourPassword',
    'login': '登陆'
}

headers = {"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
           "Accept-Encoding":"gzip, deflate, sdch, br",
           "Accept-Language":"en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4",
           "cache-control":"max-age=0",
           "Referer":"https://www.douban.com/",
           "User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
           }

r = s.post(url_login, data = formdata, headers = headers)
content = r.text
soup = BeautifulSoup(content, 'html5lib')
captcha = soup.find('img', id = 'captcha_image')
if captcha:
    captcha_url = captcha['src']
    re_captcha_id = r'<input type="hidden" name="captcha-id" value="(.*?)"/'
    captcha_id = re.findall(re_captcha_id, content)
    print (captcha_id)
    print (captcha_url) #打印验证码ｕｒｌ地址
    captcha_text = input('Please input the captcha:')　＃手工输入验证码信息
    formdata['captcha-solution'] = captcha_text
    formdata['captcha-id'] = captcha_id
    print(formdata)
    r = s.post(url_login, data = formdata, headers = headers)

r = s.get(url_contacts)

print (r.text)
#with open('contacts.txt', 'w+', encoding='utf-8') as f:
#    f.write(r.text)

如何确定form_email form_password captcha-solution  captcha-id等关键字,需要在浏览器,如chrom中F12打开调试模式,可以在Network中找到login页面，可以找到Form Data

二 使用cookie登陆:

当用户名账号方式登陆成功以后,选择记住密码,关掉网页后重新打开,在调试模式的Network里面可以找到Request Headers

里面可以看到Cookie项,得到cookie值.

#!/usr/bin/python3
#-*- coding:utf-8 -*-
import requests

url = 'http://www.douban.com'
cookies = {'cookie': 'your cookie value'}
headers = {"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
           "Accept-Encoding":"gzip, deflate, sdch, br",
           "Accept-Language":"en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4",
           "cache-control":"max-age=0",
           "Referer":"https://www.douban.com/",
           "User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
           }

r = requests.get(url, cookies = cookies, headers = headers)

获取豆瓣电影的top250:

#!/usr/bin/python3
#-*- coding:utf-8 -*-

import requests
from lxml import etree

s = requests.Session()
for id in range(0, 251, 25):
    url = 'https://movie.douban.com/top250?start=%s&filter=' % str(id)
    r = s.get(url)
    r.encoding = 'utf-8'
    root = etree.HTML(r.content)
    items = root.xpath("//ol/li/div[@class='item']")
    #print(len(items))
    for item in items:
        title = item.xpath('./div[@class="info"]//a/span[1][@class="title"]/text()') #名字
        name = title[0].encode('gb2312', 'ignore').decode('gb2312')
        print (name)
        #score = item.xpath('./div[@class="info"]/div[2]/div/span[2]/text()') #评分
        score = item.xpath('.//div[@class="bd"]//span[@class="rating_num"]/text()')[0]#评分
        print (score[0])