爬虫--拉勾网"数据分析"岗招聘信息爬取

最新推荐文章于 2021-01-15 16:18:07 发布

原创

最新推荐文章于 2021-01-15 16:18:07 发布 · 1.1k 阅读

4 ·

CC 4.0 BY-SA版权

主程序

import requests
import json
from lxml import etree
from mysqlhelper import MysqlHelper
import time
import random

class LaGouSpider():
    def __init__(self):
        # 选取热门城市
        self.sqlHelper = MysqlHelper()
        self.insertSql = '''INSERT INTO lagou VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'''
        self.headers = {
   
   
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "Referer": "https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90?px=default&city=%E4%B8%8A%E6%B5%B7",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
        }

        self.area_list = ['北京', '上海', '深圳', '广州', '杭州', '成都', '南京', '武汉', '西安', '厦门', '长沙', '苏州', '天津']
        self.index_url = "https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90?px=default&city={}"
        self.start_url = "https://www.lagou.com/jobs/positionAjax.json?"
        self.get_pageNum = ""
    def delay_time(self):
        randtime = random.randint(4,6)
        time.sleep(randtime)

    def get_cookies(self, s,city):
        response = requests.get(self.index_url.format(city), headers=self.headers)
        return response.cookies

    def