根据搜索内容爬取招聘网的职位招聘信息

最新推荐文章于 2024-01-12 23:25:49 发布

原创最新推荐文章于 2024-01-12 23:25:49 发布 · 461 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#python爬虫 #招聘网

网络爬虫专栏收录该内容

6 篇文章

订阅专栏

本文介绍了一个使用Python进行网络爬虫的实际案例，通过requests和BeautifulSoup库从51job网站抓取招聘信息，并将数据保存为CSV文件。文章详细展示了如何解析网页内容并提取关键信息。

代码：

import requests
from bs4 import BeautifulSoup
import time


def getHtml(url,code='gbk'):
    try:
        r = requests.get(url)
        r.raise_for_status()
        r.encoding = code
        return r.text
    except:
        return ""

def ParserHtml(i,htmlText):
    print("正在解析第{0}页".format(i))
    RecuitInfos = []
    soup = BeautifulSoup(htmlText,'lxml')
    #获取职位信息
    InfoPositions = soup.find_all('p',attrs={'class':'t1 '})
    #获取公司名信息
    InfoNames = soup.find_all('span',attrs={'class':'t2'})
    #获取工作地点
    InfoPlaces = soup.find_all('span', attrs={'class': 't3'})
    #获取薪资信息
    InfoSalarys = soup.find_all('span', attrs={'class': 't4'})
    #获取招聘发布条件信息
    InfoTimes = soup.find_all('span', attrs={'class': 't5'})
    for m in range(1,len(InfoPositions)):
        if (len(InfoPositions[m-1].text.split())==0)|(len(InfoNames[m].text.split())==0)|(len(InfoPlaces[m].text.split())==0)|(len(InfoSalarys[m].text.split())==0)|((InfoTimes[m].text.split())==0):
            pass
        else:
            #print(len(InfoPositions),len(InfoNames),len(InfoPlaces),len(InfoSalarys),len(InfoTimes))
            #print(InfoPositions[m-1].text.split()[0],InfoNames[m].text.split()[0],InfoPlaces[m].text.split()[0],InfoSalarys[m].text.split()[0],InfoTimes[m].text.split()[0])
            RecuitInfos.append([InfoPositions[m-1].text.split()[0],InfoNames[m].text.split()[0],InfoPlaces[m].text.split()[0],InfoSalarys[m].text.split()[0],InfoTimes[m].text.split()[0]])
    return RecuitInfos

def writeCSV(i,fw,Recruit_info):
    for Info in Recruit_info:
        print("正在写入第{0}页".format(i))
        fw.write(",".join(Info)+'\n')
    print("第{0}数据抓取完毕".format(i))

def main():
    path = 'F:'
    posttion = input("请输入要抓取的职位名称：")
    fw = open(path  +'\zhaopin_'+ posttion+'.csv', 'a+')
    row = ["职位名","公司名","工作地点","薪资","发布时间"]
    fw.write(",".join(row)+"\n")
    star_url = 'http://search.51job.com/list/000000,000000,0000,00,9,99,'
    mid_url = ',2,'
    end_url = '.html?'
    max = input("请输入最大抓取页数：")
    for i in range(1,int(max)):
        time.sleep(3)
        url = star_url + posttion +mid_url + str(i) + end_url
        htmlText = getHtml(url)
        Recruit_info = ParserHtml(i,htmlText)
        writeCSV(i,fw, Recruit_info)

if __name__ == '__main__':
    main()