基于Python网络爬虫的电子产品信息查询可视化系统

作者简介：Java领域优质创作者、优快云博客专家 、优快云内容合伙人、掘金特邀作者、阿里云博客专家、51CTO特邀作者、多年架构师设计经验、多年校企合作经验，被多个学校常年聘为校外企业导师，指导学生毕业设计并参与学生毕业答辩指导，有较为丰富的相关经验。期待与各位高校教师、企业讲师以及同行交流合作

主要内容：Java项目、Python项目、前端项目、PHP、ASP.NET、人工智能与大数据、单片机开发、物联网设计与开发设计、简历模板、学习资料、面试题库、技术互助、就业指导等

业务范围：免费功能设计、开题报告、任务书、中期检查PPT、系统功能实现、代码编写、论文编写和辅导、论文降重、长期答辩答疑辅导、腾讯会议一对一专业讲解辅导答辩、模拟答辩演练、和理解代码逻辑思路等。

收藏点赞不迷路  关注作者有好处

文末获取源码 

项目编号：BS-Python-054

一，环境介绍

语言环境：Python3.8

数据库：Mysql: mysql5.7

WEB框架：Django

爬虫：Scrapy

开发工具：IDEA或PyCharm

二，项目简介

本系统基于Python网络爬虫技术，旨在自动化收集各大电商平台上的电子产品信息，并通过数据可视化手段直观展示给用户。系统首先利用Python的爬虫框架（如Scrapy、Requests等）访问指定网站，抓取电子产品（如手机、电脑、相机等）的详细参数、价格、用户评价等关键数据。随后，利用数据处理库（如Pandas）对数据进行清洗、整理与存储。最后，通过数据可视化工具（如Matplotlib、Seaborn、Plotly或Echarts等）生成交互式图表和报告，直观展示电子产品的价格趋势、用户评分分布、热销产品排名等，帮助用户快速了解市场动态，做出更加明智的购买决策。整个系统实现了从数据抓取到分析展示的全链条自动化，提升了信息获取的效率和准确性。

三，系统展示

四，核心代码展示

#coding:utf-8
__author__ = "ila"
import base64, copy, logging, os, sys, time, xlrd, json, datetime
from django.http import JsonResponse
from django.apps import apps
from django.db.models.aggregates import Count,Sum
from .models import shoujixinxi
from util.codes import *
from util.auth import Auth
from util.common import Common
import util.message as mes
from django.db import connection
import random
from django.core.mail import send_mail
from django.conf import settings
from django.shortcuts import redirect
from django.db.models import Q
from util.baidubce_api import BaiDuBce
from .config_model import config

from dj2.settings import executor
from util.spark_func import spark_read_mysql






def shoujixinxi_register(request):
    if request.method in ["POST", "GET"]:
        msg = {'code': normal_code, "msg": mes.normal_code}
        req_dict = request.session.get("req_dict")


        error = shoujixinxi.createbyreq(shoujixinxi, shoujixinxi, req_dict)
        if error != None:
            msg['code'] = crud_error_code
            msg['msg'] = "用户已存在,请勿重复注册!"
        return JsonResponse(msg)

def shoujixinxi_login(request):
    if request.method in ["POST", "GET"]:
        msg = {'code': normal_code, "msg": mes.normal_code}
        req_dict = request.session.get("req_dict")

        datas = shoujixinxi.getbyparams(shoujixinxi, shoujixinxi, req_dict)
        if not datas:
            msg['code'] = password_error_code
            msg['msg'] = mes.password_error_code
            return JsonResponse(msg)
        try:
            __sfsh__= shoujixinxi.__sfsh__
        except:
            __sfsh__=None

        if  __sfsh__=='是':
            if datas[0].get('sfsh')!='是':
                msg['code']=other_code
                msg['msg'] = "账号已锁定，请联系管理员审核!"
                return JsonResponse(msg)
                
        req_dict['id'] = datas[0].get('id')
        return Auth.authenticate(Auth, shoujixinxi, req_dict)


def shoujixinxi_logout(request):
    if request.method in ["POST", "GET"]:
        msg = {
            "msg": "登出成功",
            "code": 0
        }

        return JsonResponse(msg)


def shoujixinxi_resetPass(request):
    '''
    '''
    if request.method in ["POST", "GET"]:
        msg = {"code": normal_code, "msg": mes.normal_code}

        req_dict = request.session.get("req_dict")

        columns=  shoujixinxi.getallcolumn( shoujixinxi, shoujixinxi)

        try:
            __loginUserColumn__= shoujixinxi.__loginUserColumn__
        except:
            __loginUserColumn__=None
        username=req_dict.get(list(req_dict.keys())[0])
        if __loginUserColumn__:
            username_str=__loginUserColumn__
        else:
            username_str=username
        if 'mima' in columns:
            password_str='mima'
        else:
            password_str='password'

        init_pwd = '123456'
        recordsParam = {}
        recordsParam[username_str] = req_dict.get("username")
        records=shoujixinxi.getbyparams(shoujixinxi, shoujixinxi, recordsParam)
        if len(records)<1:
            msg['code'] = 400
            msg['msg'] = '用户不存在'
            return JsonResponse(msg)

        eval('''shoujixinxi.objects.filter({}='{}').update({}='{}')'''.format(username_str,username,password_str,init_pwd))
        
        return JsonResponse(msg)



def shoujixinxi_session(request):
    '''
    '''
    if request.method in ["POST", "GET"]:
        msg = {"code": normal_code,"msg": mes.normal_code, "data": {}}

        req_dict={"id":request.session.get('params').get("id")}
        msg['data']  = shoujixinxi.getbyparams(shoujixinxi, shoujixinxi, req_dict)[0]

        return JsonResponse(msg)


def shoujixinxi_default(request):

    if request.method in ["POST", "GET"]:
        msg = {"code": normal_code,"msg": mes.normal_code, "data": {}}
        req_dict = request.session.get("req_dict")
        req_dict.update({"isdefault":"是"})
        data=shoujixinxi.getbyparams(shoujixinxi, shoujixinxi, req_dict)
        if len(data)>0:
            msg['data']  = data[0]
        else:
            msg['data']  = {}
        return JsonResponse(msg)

def shoujixinxi_page(request):
    '''
    '''
    if request.method in ["POST", "GET"]:
        msg = {"code": normal_code, "msg": mes.normal_code,  "data":{"currPage":1,"totalPage":1,"total":1,"pageSize":10,"list":[]}}
        req_dict = request.session.get("req_dict")

        #获取全部列名
        columns=  shoujixin