简单爬一下pc端微信小红书小程序

怎么使用：

1. 本地要新建相应txt，xlsx文件
2. 自己要准备好一些代理ip
3. 如果要搜自己指定的关键字的话，pararm里的关键字（keyword）要改一下，还有get_X_sign里也要对应改一下，就是关键字要改成，‘%E4%BA%BA%E7%94%9F%E6%84%9F%E6%82%9F’同样的形式
4. 时间久了的话，一些参数会过期，重新抓包获取参数

其他：

参考自:
https://blog.youkuaiyun.com/m0_46639364/article/details/108012594

代码：

import time
import requests
import re
import urllib3
from selenium.webdriver import Chrome
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import json
from openpyxl  import load_workbook
import hashlib
class xhs_class(object):
    def __init__(self):
        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
        self.url = 'https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/search/notes?'
        self.key_dict=dict()
        with open('小红书url.txt', 'r', encoding='utf-8')as f:
            r = f.read().replace('\ufeff', '')
            self.old_list = r.split('\n')
            #print(self.old_list)
        options = Options()
        options.add_argument('--headless')
        self.driver = Chrome(options=options)
    def get_X_sign(self):
        for i in range(1,6):
            par = {
                        'keyword': '美食',
                        'sortBy': 'general',
                        'page': i,
                        'pageSize': '20',
                        'prependNoteIds':'',
                        'needGifCover': 'true',
                    }
            qwq= '/fe_api/burdock/weixin/v2/search/notes?keyword='+'%E4%BA%BA%E7%94%9F%E6%84%9F%E6%82%9F'+'&sortBy=general&page='+str(i)+'&pageSize=20&prependNoteIds=&needGifCover=true'
            s=qwq+'WSUDD'
            #print (s)
            md5 = hashlib.md5()
            md5.update(s.encode("utf-8"))
            X_sign='X'+md5.hexdigest()
            #print(X_sign)
            self.key_dict[str(i)]=X_sign
        print(self.key_dict)
    def get_ip(self,num):
        res=requests.get('http://webapi.http.zhimacangku.com/getip?num=200&type=1&pro=&city=0&yys=0&port=1&time=1&ts=0&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1&regions=')
        #print(res.text)
        http_dict=dict()
        a=res.text.split('\r\n')
        a.pop()
        for i, item in enumerate(a):
            lst=a[i].split(':')
            key=lst[0]
            value=lst[1]
            http_dict[key]=value
        target=0
        for key,value in http_dict.items():
            proxyHost = key
            proxyPort = value
            proxyMeta = "http://%(host)s:%(port)s" % {
                "host" : proxyHost,
                "port" : proxyPort,
            }
            self.proxies = {
                "http"  : proxyMeta,
                "https"  : proxyMeta
            }
            target+=1
            if target==num:
                break
        return self.proxies
    def get_detail_url(self):
        self.get_X_sign()
        self.num=0
        for key, value in self.key_dict.items():
            headers = {
                'Host': 'www.xiaohongshu.com',
                'Connection': 'keep-alive',
                'Authorization': 'wxmp.c3657635-30c6-421c-88e8-d76d1676ae12',
                'Device-Fingerprint': 'WHJMrwNw1k/Hg/nCupWeBd8sBskqluBSRBmzIRRI207qeIoxRiCuJhSk7OZmoaFLmYPM6m5bBIGFGAOhuNZ6DRl6kLx8sRQcTdCW1tldyDzmauSxIJm5Txg==1487582755342',
                'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat',
                'X-Sign': value,
                'content-type': 'application/json',
                'Referer': 'https://servicewechat.com/wxb296433268a1c654/60/page-frame.html',
                'Accept-Encoding':'gzip, deflate, br',
                
            }
            params = {
                    'keyword': '人生感悟',
                    'sortBy': 'general',
                    'page': key,
                    'pageSize': '20',
                    'prependNoteIds':'',
                    'needGifCover': 'true',
                }
            res= requests.get(self.url,headers=headers,params=params,verify=False,proxies=self.get_ip(self.num))
            self.num+=1
            #print (self.get_ip(self.num))
            res2=res.text
            #print(res)
            #print(res.status_code)
            res_dict = json.loads(res2)
            #print(res_dict)
            notes = res_dict['data']['notes']
            for note in notes:
                    id = note['id']
                    #print(id)
                    self.detail_url = 'https://www.xiaohongshu.com/discovery/item/' + id
                    #print(self.detail_url)
                    if self.detail_url in self.old_list:
                        print('链接已存在。')
                        continue
                    else:
                        with open('小红书url.txt', 'a', encoding='utf-8')as w:
                            w.write('\n')
                            w.write(self.detail_url)
                        self.get_detail()
                        self.store_data()
                        continue

    def get_detail(self):
        proxy = self.get_ip(self.num)
        proxy=proxy['http']
        # 设置代理IP
        ops = webdriver.ChromeOptions()
        ops.add_argument('--proxy-server=%s' % proxy)
        self.driver= webdriver.Chrome(chrome_options=ops)
        ops.add_argument('--headless')
        self.driver.get(self.detail_url)
        self.num+=1
        #print (self.chrome)
        time.sleep(1.5)
        try:
            self.pub_time = str(self.driver.find_element_by_xpath('//div[@class="publish-date"]/span').text).replace(
                '发布于', '')
            print(self.pub_time)
        except:
            self.pub_time = ' '
        try:
            self.author = self.driver.find_element_by_class_name('name-detail').text
            print(self.author)
        except:
            self.author = ' '
        try:
            self.title = self.driver.find_element_by_class_name('title').text
            print(self.title)
        except:
            self.title=' '
        try:
            self.content = self.driver.find_element_by_class_name('content').text
            print(self.content)
        except:
            self.content=' '
        
    def store_data(self):
        wb = load_workbook('数据表.xlsx')
        sheet = wb.get_sheet_by_name("Sheet1")
        excel_head=['作者','发布时间','标题','内容']
        for i, item in enumerate(excel_head):
            sheet.cell(row = 1,column=i+1,value=item)
        data=[self.author,self.pub_time,self.title,self.content]
        print(data)
        sheet.append(data)
        wb.save('数据表.xlsx')
        print('保存成功'+str(self.num))
if __name__ == '__main__':
    xhs = xhs_class()
    xhs.get_detail_url()