python爬取相关话题论文的相关信息

# encoding = 'utf-8'

import requests
from bs4 import BeautifulSoup
import pandas as pd


url = 'https://search.cnki.com.cn/Search/Result'

all_data = []

for page in range(50):
    print("page:", page)
    data = {'searchType':'MulityTermsSearch',
           'Theme':'science',
            'Page':page}

    headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'}
    response = requests.post(url, headers=headers, data=data)
    soup = BeautifulSoup(response.text, 'html.parser')
    cc = soup.find_all(name='div',attrs={"class":"list-item"})

    for i in range(1, len(cc)):
        try:
            down_url = cc[i].find('a')['href'][2:]
            title = cc[i].find('a')['t