使用requests或scrapy爬取拉勾动态数据

由于本人不怎么会组织语言讲解， 所以就直接上代码了，见谅,不会的可以问我。

1. requests爬取

import requests
import time

headers = {
    'Referer': 'https://www.lagou.com/jobs/list_python%E7%88%AC%E8%99%AB?labelWords=sug&fromSearch=true&suginput=python',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36'
}
session = requests.session()  # session会话保持
session.headers.update(headers)  # 更新headers
# get请求的地址为headers里面的 “Referer”的地址
session.get('https://www.lagou.com/jobs/list_python%E7%88%AC%E8%99%AB?labelWords=sug&fromSearch=true&suginput=python')
data = {
    'first': 'false',  # 第一页为true  这里统一写为false
    'pn': 1,    # 爬取多少页， 这里自己修改
    'kd': 'python爬虫'  # 这里修改为自己想要爬取的内容
}

'''
https://www.lagou.com/jobs/companyAjax.json
https://www.lagou.com/jobs/positionAjax.json

地址注意不要写错了
'''
r1 = session.post(url='https://www.lagou.com/jobs/positionAjax.json?city=%E5%8C%97%E4%BA%AC&needAddtionalResult=false',
                  headers=headers,
                  data=data)
# print(r1.json())
data = r1.json()
for result in data['content']['positionResult']['result']:  # json数据提取
    print(result['positionName'].strip())  # 爬取​职位名称
    print(result['salary'])  # 爬取职位薪资
    time.sleep(1)

2. scrapy爬取

import time

import scrapy
import json

class LagouSpider(scrapy.Spider):
    name = 'lagou'
    allowed_domains = ['www.lagou.com']
    # start_urls = ['http://www.lagou.com/']

    # 重写 start_requests 方法
    def start_requests(self):
        headers = {
            'Referer': 'https://www.lagou.com/jobs/list_python%E7%88%AC%E8%99%A'
                       'B?labelWords=sug&fromSearch=true&suginput=python',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKi'
                          't/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36'
        }
        yield scrapy.Request(
            'https://www.lagou.com/jobs/list_python%E7%88%AC%E8%99%AB?labelWords=sug&fromSearch=true&suginput=python',
            # meta={'cookiejar': 1} 开启cookie记录
            headers=headers, meta={'cookiejar': 1}, callback=self.do_post)

    def do_post(self, response):

        # 响应cookie
        Cookie1 = response.headers.getlist('Set-Cookie')
        # print(Cookie1)
        # print(response.text)
        # city=****  这里是选择的城市 可以去掉也就是全国 可以在浏览器中测试一下
        post_url = 'https://www.lagou.com/jobs/positionAjax.json?city=%E5%8C%97%E4%BA%AC&needAddtionalResult=false'  
        
        headers = {
            'Referer': 'https://www.lagou.com/jobs/list_python%E7%88%AC%E8%99%AB?'
                       'labelWords=sug&fromSearch=true&suginput=python',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
                          ' (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36', }
        for i in range(1, 7):  # 这里我只爬取了6页 
            data = {
                'first': 'false',
                'pn': str(i),
                'kd': 'python爬虫'
            }
            yield scrapy.FormRequest(
                                 url=post_url,
                                 formdata=data, headers=headers,
                                 # 携带cookie
                                 meta={'cookiejar': response.meta['cookiejar']},
                                 callback=self.parse)

    def parse(self, response):
        # 请求cookie
        Cookie2 = response.request.headers.getlist('Cookie')
        # print(Cookie2)
        # print(response.text)

        # 将文本解析为json格式
        data = json.loads(response.text)
        for result in data['content']['positionResult']['result']:  # json数据提取
            print(result['positionName'].strip())
            print(result['salary'])
            time.sleep(1)

可以拿最新的腾讯招聘练练手 哈哈