爬虫实战19：爬取tandfonline.com文章信息

shifanfashi

于 2019-07-14 17:30:36 发布

阅读量2.1k

点赞数 1

CC 4.0 BY-SA版权

分类专栏：爬虫实战爬虫实战文章标签：爬虫实战

本文链接：https://blog.youkuaiyun.com/shifanfashi/article/details/95902306

import requests
import bs4
import xlwt
import openpyxl


class aticle_title():
    def __init__(self):
        self.url = "https://www.tandfonline.com/action/doSearch?AllField=urban+design&Ppub=%5B20151107+TO+20161107%5D&content=standard&countTerms=true&target=default&sortBy=&pageSize=50&subjectTitle=&startPage="
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
        }

    def get_title(self, page):
        response = requests.get(self.url+str(page), headers=self.headers,  timeout=10).text

        table = bs4.BeautifulSoup(response, "lxml").find('div', {'id': 'allTabsContainer'}).find('ol', {'class': 'search-results'}).find_all('li')
        f = open('result.txt', 'a+', encoding='utf-8')
        for item in table:
            title = item.find('article').get_text('data-title')
            f.write(title + '\n')
        f.c