python网络爬虫实战之爬取OJ所有问题信息

更多爬虫内容请关注博主的专栏python3网络爬虫

import requests
from bs4 import BeautifulSoup
import re

'''
    @author: Face_to_sun
    @modify:2018-12-01

'''
def getHTMLText(url, page):
    headers = {
        'Host': 'acm.cug.edu.cn',
        'Referer': 'http://acm.cug.edu.cn/',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36'
    }
    data = {
        'page': str(page)
    }
    try:
        url = url + str(page)
        response = requests.get(url, headers=headers, data=data)
        response.raise_for_status();
        response.encoding = 'utf8'
        print(url)
        return response.text
    except:
        return "产生异常"

# def getHead(soup):
#     thead = soup.find_all(name='tr', class_='toprow')
#     for tr in thead:
#         ths = tr.find_all('th')
#         for th in ths:
#             print(th.string)

def getBody(soup):
    problem = {}
    # table = soup.find_all(name='table', id=problemset)
    table = soup.find_all(id="problemset")
    for i in table:
        tbody = i.find_all('tbody')
    for body in tbody:
        trs = body.find_all('tr')
        for tr in trs:
            tds = tr.find_all('td')
            problem['Bool'] = tds[0].get_text()
            problem['problem ID'] =tds[1].get_text()
            problem['Title'] = tds[2].get_text()
            problem['Source/Category'] = tds[3].get_text()
            problem['AC'] = tds[4].get_text()
            problem['Submit'] = tds[5].get_text()
            if int(tds[5].get_text()) == 0:
                problem['Accuracy rate'] = 0
            else:
                problem['Accuracy rate'] = int(tds[4].get_text())/int(tds[5].get_text())
            print(problem)

if __name__ == "__main__":
    url = 'http://acm.cug.edu.cn/problemset.php?page='
    for i in range(1,13):
        html = getHTMLText(url, i)
        # with open('problem.html', 'w', encoding='utf8') as f:
        #     f.write(html)
        #error: python编码问题——解决python3 UnicodeEncodeError: 'gbk' codec can't encode character '\xXX' in position XX
        soup = BeautifulSoup(html,'lxml')
        # getHead(soup)
        getBody(soup)

运行结果