给我爬！10-22

最新推荐文章于 2025-12-17 15:36:40 发布

原创最新推荐文章于 2025-12-17 15:36:40 发布 · 186 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#python #开发语言 #后端

该博客介绍了Python网络爬虫的基础操作，包括使用requests库发送GET和POST请求，处理响应状态码、编码问题，下载网页内容和图片，以及设置User-Agent和SSL证书。同时，展示了如何处理请求参数、表单数据，以及使用cookies进行会话管理。

import urllib3
import requests
import json
def write_to_file(filename, html):
f=open(filename,‘w’,encoding=‘utf-8’);
f.write(html);
f.close;

def load_page_with_urllib(filename,res):
html = res.read().decode(‘utf-8’)
print(html)
write_to_file(filename, html)

def load_page_with_requests(filename,res):
html = res.content.decode(‘utf-8’)
print(html)
write_to_file(filename, html)

def send_url():
url=“http://httpbin.org/get”
res=requests.get(url)
print(res.status_code)
print(res.encoding)
print(res.headers)
print(res.text)
load_page_with_requests(‘send_url.html’, res)
print(“Host:”+res.json()[‘headers’][‘Host’])

def send_url_with_pic():
url=“https://www.baidu.com/img/PCtm_d9c8750bed0b3c7d089fa7d55720d6cf.png”
res=requests.get(url)
print(res.status_code)
with open(‘kkk.png’,‘wb’) as f:
f.write(res.content)
f.close()

def send_with_text_encoding():
url=“http://www.google.cn/”
res=requests.get(url)
print(res.headers.get(“Content-Type”))
print(res.encoding)
print(res.apparent_encoding)
print(res.text)
res.encoding=res.apparent_encoding
print(res.text)

def send_with_text_encoding_baidu():
url=“https://www.baidu.com/”
headers={
“User-Agent”:“Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36”
}
res=requests.get(url, headers=headers)
print(res.headers.get(“Content-Type”))
print(res.encoding)
print(res.apparent_encoding)
print(res.text)

def send_get_with_param():
url=“http://cn.bing.com/search”
word = {“q”:“Python网络爬虫”}
headers = {
“User-Agent”: “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36”
}
res=requests.get(url,params=word,headers=headers)
load_page_with_requests(‘send_get_with_search.html’,res)

def send_post_with_form():
url=“http://fanyi.youdao.com/translate?smartresult=dic&client=fanyideskweb”;
headers = {
“User-Agent”: “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36”
}
formdata = {
“i”: “我觉得你人不错”,
“from”: “AUTO”,
“to”: “AUTO”,
“smartresult”: “dict”,
“client”: “fanyideskweb”,
“salt”: “16348709989680”,
“sign”: “dae5fd6b6f4221ca9cda39bb46c9bfec”,
“lts”: “1634870998968”,
“bv”: “7b07590bbf1761eedb1ff6dbfac3c1f0”,
“doctype”: “json”,
“version”: “2.1”,
“keyfrom”: “fanyi.web”,
“action”: “FY_BY_REALTlME”
}
res=requests.post(url, data=formdata, headers=headers)
print(res.status_code)
print(res.text)
print(“翻译前：”+res.json()[‘translateResult’][0][0][‘src’])
print(“翻译后：”+res.json()[‘translateResult’][0][0][‘tgt’])

def send_with_ssl():
url=“https://kyfw.12306.cn/otn/leftTicket/init”
#res001=requests.get(url)
#print(res001.status_code)
#print(res001.text)

#res002=requests.get(url, verify=False)
#print(res002.status_code)
#print(res002.text)

res003 = requests.get(url, verify="D:/temp.txt")
print(res003.status_code)
print(res003.text)

def send_with_cookies():
url=“http://httpbin.org/cookies”
headers = {
“User-Agent”: “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36”,
“Cookie”:“OUTFOX_SEARCH_USER_ID = -1942405773 @ 10.108.160.101;JSESSIONID = aaaUTOVsl72cZgmRt4LYx;OUTFOX_SEARCH_USER_ID_NCOO = 710165529.6036291;fanyi - ad - id = 118539;fanyi - ad - closed = 1;___rl__test__cookies = 1634870989954”
};

res001=requests.get(url,headers=headers)
print(res001.status_code)
print(res001.text)

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36"
    #"Cookie":"OUTFOX_SEARCH_USER_ID=-1942405773@10.108.160.101; JSESSIONID=aaaUTOVsl72cZgmRt4LYx; OUTFOX_SEARCH_USER_ID_NCOO=710165529.6036291; fanyi-ad-id=118539; fanyi-ad-closed=1; ___rl__test__cookies=1634870989954"
};
cookies={
    "Cookie": "JSESSIONID=aaaUTOVsl72cZgmRt4LYx; OUTFOX_SEARCH_USER_ID_NCOO=710165529.6036291; fanyi-ad-id=118539; fanyi-ad-closed=1; ___rl__test__cookies=1634870989954"
}
res002 = requests.get(url, headers=headers,cookies=cookies)
print(res002.status_code)
print(res002.text)

if name ==‘main’:
send_url()
send_url_with_pic()
send_with_text_encoding()
send_with_text_encoding_baidu()
send_get_with_param()
send_post_with_form()
#send_with_ssl()
send_with_cookies()