django批量导入数据

最新推荐文章于 2025-06-12 09:09:00 发布

CHJXBT

最新推荐文章于 2025-06-12 09:09:00 发布

阅读量1.8k

点赞数

CC 4.0 BY-SA版权

分类专栏：代码文章标签：代码

本文链接：https://blog.youkuaiyun.com/sinat_30812239/article/details/84587680

代码专栏收录该内容

1 篇文章

订阅专栏

本文介绍如何使用Django框架结合Python爬虫技术抓取并解析网页数据，重点讲解了去重处理、数据下载及模型批量创建的过程。通过实例演示了如何设置Django环境、调用API获取数据、解析JSON响应、图片下载保存以及数据库操作。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

# coding:utf-8
import json
import urllib.request
import time
import requests
import os
from django.conf import settings

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dongliang.settings")

'''
Django 版本大于等于1.7的时候，需要加上下面两句
import django
django.setup()
否则会抛出错误 django.core.exceptions.AppRegistryNotReady: Models aren't loaded yet.
'''
import django


if django.VERSION >= (1, 7):  # 自动判断版本
    django.setup()

from user.models import User
from operation.models import CourseComments

headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'
}

response = requests.get('https://www.yojiang.cn/api/lesson/comment/query?series_id=67', headers=headers)
json_str = response.content.decode()
comment_items = json.loads(json_str)
data = comment_items['data']['items']

# 实现去重
new_data = []
new_data.append(data[0])
for dic in data:
    # print len(l4)
    # print('dic===', dic)
    k = 0
    for item in new_data:
        # print('item===', item)
        # print(dic['user']['nickname'])
        # print(item['user']['nickname'])
        # print 'item'
        if dic['user']['nickname'] != item['user']['nickname']:
            # print('名字不同')
            k = k + 1
        else:
            # print('名字相同')
            break
        if k == len(new_data):
            # print('今日')
            new_data.append(dic)

# user_list = []
for i in new_data:
    print(i['user']['nickname'])
    img_src = i['user']['portrait']
    img_name = img_src.split('/')[-2]
    t = time.localtime()
    path = os.path.join(settings.BASE_DIR, 'media/user/%d/%02d/%s.jpg' % (t.tm_year, t.tm_mon, img_name))
    # print(path)
    # 将远程数据下载到本地，第二个参数就是要保存到本地的文件名
    result = urllib.request.urlretrieve(img_src, path)
    file_name = result[0].split('/media/')[1]
#     # print(file_name)
#     user_list.append(User(username=i['user']['nickname'], user_avatar=file_name, is_active=False))
# User.objects.bulk_create(user_list)

# comment_list = []
# for i in new_data[:47]:
#     user = User.objects.filter(username=i['user']['nickname'])
#     comment_list.append(CourseComments(user_id=user[0].id, courses_id=1, comments=i['content'], status=0))
# CourseComments.objects.bulk_create(comment_list)
#
# for i in new_data[:47]:
#     comment = CourseComments.objects.filter(comments=i['content']).first()
#     comment.create_time = i['create_time']
#     comment.save()

print('完成')