Python自动爬取拉钩教育课程

本课程纯属学习交流，如有侵权，请告知，立马删除

import requests
import json
import csv
import time
import random
# import shlex

# print(random.randint(1,17))
# exit()

courseId = "1"  # 想学哪一门，改这里

url = "https://*****/getCourseLessons?courseId=" + courseId
payload = {}
headers = {
  'authority': '*',
  'pragma': 'no-cache',
  'cache-control': 'no-cache',
  'accept': 'application/json, text/plain, */*',
  'x-l-req-header': '{deviceType:1}',
  'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36',
  'origin': 'https://',
  'sec-fetch-site': 'same-site',
  'sec-fetch-mode': 'cors',
  'sec-fetch-dest': 'empty',
  'referer': 'https://',
  'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
  'cookie': ''
}

response = requests.request("GET", url, headers=headers, data = payload)

# courseInfo = response.json()

data = []


courseList = json.loads(response.text).get('content').get('courseSectionList')
for course in courseList:
  for cl in course['courseLessons']:
    if not cl['audioMediaDTO']:
      continue
    data.append([courseId,str(cl['id']),cl['theme'],cl['audioMediaDTO']['fileUrl'],str(cl['audioMediaDTO']['fileId']),cl['videoMediaDTO']['fileUrl'],str(cl['videoMediaDTO']['fileId']),''])
    pass
  pass
rows = []
for x in data:
  time.sleep(random.randint(5,50))
  url = "https://****.com/v1/**/***/getCourseLessonDetail?lessonId="+x[1]
  try:
    response1 = requests.request("GET", url, headers=headers, data = payload)
    pass
  except:
    print("except")
    continue
    raise
  contents = json.loads(response1.text).get('content').get('textContent')
  title = json.loads(response1.text).get('content').get('theme')
  with open('***/python/***/'+title+'.html','w') as f:#使用with open()新建对象f
    for i in contents:
        f.write(i+'\n')
  pass

print(1)

以上只是举个例子，抓取网页啥的都可以使用的.

欢迎评论区讨论