代理IP,模拟登陆，request.get设置cookie

最新推荐文章于 2023-12-01 20:15:00 发布

Hello_Jandy

最新推荐文章于 2023-12-01 20:15:00 发布

阅读量4.6k

点赞数

分类专栏：爬虫

本文链接：https://blog.youkuaiyun.com/weixin_42141853/article/details/80821642

版权

爬虫专栏收录该内容

17 篇文章

订阅专栏

本示例展示了如何使用Python爬取优快云博客的文章列表及具体内容，包括设置请求头、Cookie、代理IP等步骤，并实现了随机访问博客文章。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import random

import re
import requests

#设置请求头
import time

headers = {
    'Host': 'https://blog.youkuaiyun.com/weixin_42141853',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
    'Accept-Encoding': 'gzip, deflate',
    'Referer': 'http://www.baidu.com',
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
}

#设置Cookie模拟登陆
cookie={'Cookie':'uuid_tt_dd=9355752249176600242_20180327; __yadk_uid=6BHVQvWYnoNxl3eLA9VqAKJDzcTsHk7N; kd_user_id=fa963059-44ce-48f6-9b11-db28f9c2f8dc; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=1788*1*PC_VC; UM_distinctid=1642bb7fd891ff-084f74390cc1c7-28792744-15aea-1642bb7fd8a1a7; CNZZDATA1272993679=430637425-1529739593-https%253A%252F%252Fblog.youkuaiyun.com%252F%7C1529739593; is_advert=Mon%2C%2015%20Jan%202255%2008%3A42%3A15%20GMT; TY_SESSION_ID=d524b848-bfa3-48df-9415-7da43ebcbfd0; ADHOC_MEMBERSHIP_CLIENT_ID1.0=923d341d-8a44-6b42-7269-aa78de1749a8; UserName=weixin_42141853; UserInfo=kWcYNbZy0%2B6z%2F%2BsM24JwP04fLQd0exodlLoFEsce8n7445vV4LzD9sBOWCu17CgAWLDaZFDI8e3ZRa8B6o0G8A%3D%3D; UserNick=Hello_Jandy; UN=weixin_42141853; AU=B92; BT=1530015086665; smidV2=201806261921224990b452919eb712d7a8052f6fa74e4c0015cd7a766149570; dc_session_id=10_1530002420268.196586; dc_tos=paxjbz; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1529758375,1529826531,1529887665,1530002525; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1530015264'
}

#设置代理IP
proxyLists = ['111.231.115.150:8888', '183.33.151.102:808','115.204.29.26:6666','118.190.95.269001:9001', '175.0.72.205:61202', '60.216.177.152:8118',
              '101.236.35.98:8866']
proxies ={'http': proxyLists[random.randrange(len(proxyLists))]}
url='https://blog.youkuaiyun.com/weixin_42141853'
data=requests.get(url,headers,cookies=cookie,proxies=proxies).text
pat='href="(https://blog.youkuaiyun.com/weixin_42141853/article/details.*?)"'
articalLists=re.findall(pat,data)
for i in range(1000):
    time.sleep(2)
    #文章路由
    articalurl = articalLists[random.randrange(len(articalLists))]
    #代理IP
    proxies = {'http': proxyLists[random.randrange(len(proxyLists))]}
    data = requests.get(articalurl, headers, cookies=cookie, proxies=proxies).text
    print(data)
    print(articalurl)
    print(proxies)
    print(i)