Python3~scrapy项目之爬取当前页和下一页

最新推荐文章于 2025-05-28 11:15:24 发布

Arthur54271

最新推荐文章于 2025-05-28 11:15:24 发布

阅读量1w

点赞数 2

CC 4.0 BY-SA版权

分类专栏： Python3 爬虫 scrapy xpath 文章标签： scrapy 爬虫

本文链接：https://blog.youkuaiyun.com/zbrj12345/article/details/80514425

Python3 同时被 3 个专栏收录

104 篇文章

订阅专栏

爬虫

40 篇文章

订阅专栏

scrapy

11 篇文章

订阅专栏

本文介绍了一个使用Scrapy框架爬取腾讯官网职位信息的Python3项目，包括当前页和下一页的数据。通过解析网页HTML，提取职位的日期、地点、人数、类型、名称和链接，并进一步请求详情页获取工作职责和要求。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

# -*- coding: utf-8 -*-
import scrapy
from urllib import request
from Py06_2018_3_16.items import TencentItem

class tencentNextPageSpider(scrapy.Spider):
    name = 'tencent_next_page'
    allowed_domains = ['hr.tencent.com']
    start_urls = ['https://hr.tencent.com/position.php?&start=0']
    # base_url='https://hr.tencent.com/position.php?&start=%s'
    # for i in range(0,391):
    #     # url=base_url%(i*10)
    #     start_urls.append(url)

    def parse(self, response):
        job_even=response.xpath('//tr[@class="even"]')
        job_odd=response.xpath('//tr[@class="odd"]')
        #合并数组
        jobs=job_even+job_odd

        for job in jobs:
            print(job)
            item=TencentItem()
            #时间
            date=job.xpath('.//td[5]/text()').extract()[0]
            item['date']=date
            #地点
            location=job.xpath('.//td[4]/text()').extract()[0]
            item['location']=location
            #人数
            num = job.xpath('.//td[3]/text()').extract()[0]
            item['num'] = num
            #职位类别
            type = job.xpath('.//td[2]/text()').extract()#['XXX']
            print(type)
            type=self.getvalue(type)

            item['type'] = type

            #职位名称
            name=job.xpath('.//td[1]/a/text()').extract()[0]
            item['name']=name
            #链接
            url = job.xpath('.//td[1]/a/@href').extract()[0]
            # 比较低级
            # url='https://hr.tencent.com/'+url
            #高级  #拼接全路径
            url = request.urljoin(response.url,url)
            item['url'] = url
            # print(name+'\t'+type+'\t'+num+'\t'+location+'\t'+date+'\t'+url)
            print(url)
            print('~~~~~~~~~~~')

            # yield item
            #请求详情页，二级流程
            yield scrapy.Request(url=url,callback=self.parse_detail,meta={'data':item})

        #下一页
        next_url=response.xpath('//a[@id="next"]/@href').extract()[0]
        next_url=request.urljoin(response.url,next_url)

        #是否是最后一页
        last_page=response.xpath('//a[@id="next"]/@class').extract()
        last_page=self.getvalue(last_page)
        if last_page!='noactive':
            yield scrapy.Request(url=next_url,callback=self.parse)

    def parse_detail(self,response):
        # print('~~~~~~~~~~~detail~~~~~~~~~~~~`')
        # print(response.text)
        item=response.meta['data']
        with open('detail.html','w',encoding='utf-8') as f:
            f.write(response.body.decode('utf-8'))
        #工作职责
        duty=response.xpath('//tr[@class="c"][1]//li/text()').extract()
        # print(duty)#字符串列表
        duty=''.join(duty)
        item['duty'] = duty
        #工作要求
        rq=response.xpath('//tr[@class="c"][2]//li/text()').extract()
        rq=''.join(rq)
        item['rq']=rq
        # print(duty+'\n')
        # print(rq)
        # print('~~~~~~~~~~~~~~~`')
        yield item

    def getvalue(self,value):

        # return value[0] if value else ''
        if value == []:
            return ''
        else:
            return value[0]