惠农网_破译反爬虫_使用代理IP爬取数据(源码)_一蓑烟雨任平生

费话不多说直接上代码

今天要倒霉的网站是惠农网行情大厅

import requests
from bs4 import BeautifulSoup  # 用来解析网页
import uuid
import pymysql
import time  # 导入时间隔
import random
from fake_useragent import UserAgent

cookk = {
   
   
    'Cookie': 'sessionId=S_0KHA8RH66NM124OL; Hm_lvt_91cf34f62b9bedb16460ca36cf192f4c=1604907974,1604908122; Hm_lpvt_91cf34f62b9bedb16460ca36cf192f4c=1604912363; deviceIdRenew=1; lmvid=1e19526d8b71a2ee4a3a8ff527100d30; lmvid.sig=hV1QMMdZFpUaz_-YYY93w; deviceId=05ac3ef-5c72-405f-be76-83e97e186; hnUserTicket=d67ed8dc-df8a-45fe-85cd-1530efae920d; hnUserId=125807455'}

headers = {
   
   
    'Host': 'www.cnhnb.com',
    'Referer': 'https://www.cnhnb.com/hangqing/cdlist-0-0-0-0-0-1/',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',

}
types = [2003191, 2003192, 2003193, 2003194, 2003195, 2003196, 2003197, 2003198, 2003200]


def func():
    conn = pymysql.connect(host='127.0.0.1', user='root', passwd='123456', db='zhang', charset='utf8')
    cur = conn.cursor()
    print("数据库已连接")
    for type in types:
        for i in range(19, 50):