|
|
（一）Selenium自动化Python爬虫工具采集漫画视频等约10万条存入.csv文件作为数据集； （二）使用pandas+numpy或MapReduce对数据进行数据清洗，生成最终的.csv文件并上传到hdfs； （三）使用hive数仓技术建表建库，导入.csv数据集； （四）离线分析采用hive_sql完成，实时分析利用Spark之Scala完成; （五）统计指标使用sqoop导入mysql数据库； （六）使用Flask+echarts进行可视化大屏开发； （七）使用机器学习、深度学习的算法进行个性化漫画视频推荐； （八）使用卷积神经网络KNN、CNN实现漫画视频流量预测； （九）搭建springboot+vue.js前后端分离web系统进行个性化推荐界面、漫画视频流量预测界面、知识图谱等实现； |
核心算法代码分享如下：
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=Edge，chrome=1">
<meta name="renderer" content="webkit">
<title>Document</title>
<link rel="stylesheet" type="text/css" href="/static/css/cao.css">
<script src="/static/js/jquery.min.js"></script>
<script src="/static/js/echarts.min.js"></script>
<script src="/static/js/dark.js"></script>
<script src="/static/js/echarts-wordcloud.min.js"></script>
</head>
<body>
<div class="header">
<h2 style="font-Size:30px">bilibili热门视频大数据分析平台</font></h2>
<h2 style="font-Size:30px">hadoop+spark+hive离线与实时计算一体化大数据平台(可选装推荐系统、预测系统、知识图谱)</font></h2>
</div>
<div class="main clearfix">
<div class="main-left">
<div class="border-container">
<div id="rank"></div>
<script>
var barq5 = echarts.init(document.getElementById("rank"))
$.ajax({
type:"GET",
url:"/table02",
dataType:"json",
success:function(data){
nbDatad(data)
}
})
function nbDatad(data){
var barq5Values = []
var barq5Names = []
for(var i =0;i< data.length;i++){
barq5Values.push(data[i].num)
barq5Names.push(data[i].name)
}
const options = {
title:{
text:"最热视频分类Top10",
left:20,
top:20,
textStyle:{
fontSize:30,
color:" #b300ff"
},
},
tooltip:{
show:true
,textStyle:{
fontSize:20,
color:"#00ffff",
},
formatter: "分类:{b} 个数:{c}"
},
grid:{
top:'20%',
left:'3%',
right:"6%",
bottom: "3%",
containLabel:true
},
xAxis:{
type:"category",
data:barq5Names,
axisLabel: {
textStyle: {
color: "#00ffff"
},
fontSize:16//调整坐标轴字体大小
},
},
yAxis:{
type:"value",
axisLabel: {
textStyle: {
color: '#00ffff'
},
fontSize:25//调整坐标轴字体大小
}
},
textStyle: {
color: '#51ffe2',
fontSize :20
},
series:[
{
type:"bar",
data:barq5Values,
smooth: true,
itemStyle: {
color: "#99CCFF",
barBorderRadius:[88,88,0,0],
width: 20,
},
markPoint: {
data: [
{ type: 'max', name: 'Max' },
{ type: 'min', name: 'Min' }
],
symbol:'image://http://img30.360buyimg.com/poprx/jfs/t23662/172/668395134/12944/5653b1ff/5b3c2737N3d867c2f.png',
symbolSize:[80, 80],
symbolOffset:[0,-30],//位置偏移
label: {
textStyle:{
fontSize:40,
color:"#0051ff"
},}
}
}
]
}
barq5.setOption(options)
}
</script>
<ul class="three-pie clearfix">
<div id="pao"></div>
<script>
var msu = echarts.init(document.getElementById("pao"))
$.ajax({
type:"GET",
url:"/table01",
dataType:"json",
success:function(data){
const msuwddata = data.map(function(item){
return {
name:item.name,
value:item.num,
}
})
const leidaoptions = {
title:{
text:"bilibili热门视频话题Top10",
left:20,
top:20,
textStyle:{
fontSize:30,
color:"#b300ff",
}
},
tooltip:{
show:true
,textStyle:{
fontSize:30,
color:"#00ffff",
},
formatter: "标签:{b} 数量:{c} 占比{d}%"
},
legend: {
top: '85%',
left: 'center',
textStyle:{
fontSize:30,
color:"#ff",
}
},
color: ['#00FFFF','#0066FF','#9900FF ','#FF69B4 ','#FFA07A','#66CCFF','#99CC66','#CCFFFF ',' #00FF99 ','#008888'],
series: [
{
name: '比例',
radius: ['40%', '70%'],
// roseType: 'radius',
min: 756,
max:810,
type:"pie",
width: '50%',
data:msuwddata,
label: {
show: false,
position: 'center',
},
}
]
}
msu.setOption(leidaoptions)
}
})
</script>
</ul>
<span class="top-left border-span"></span>
<span class="top-right border-span"></span>
<span class="bottom-left border-span"></span>
<span class="bottom-right border-span"></span>
</div>
</div>
<div class="main-middle">
<div class="border-container">
<div id="map"></div>
<script>
var loudouEcharts = echarts.init(document.getElementById("map"))
$.ajax({
type:"GET",
url:"http://127.0.0.1:5000/table06",
dataType:"json",
success:function(getJson){
console.log(getJson)
var specialized_in = [];
var num = [];
var i=0;
var data=[]
for(var v in getJson){
// Data.push(data[i])
//console.log(v)
//console.log(getJson[v].title)
specialized_in.push(getJson[v].title)
num.push(getJson[v].shares)
item={}
item.name=getJson[v].title
item.value=getJson[v].shares
data.push(item)
i=i+1
// var name = [];
// var Chargeback = [];
// for(var i = 0; i < 10;i++){
// // Data.push(data[i])
// name.push(data.name[i])
// Chargeback.push(data.Chargeback[i])
}
const dataOption = {
title:{
text:"热门视频分享量Top10",
left:20,
top:20,
textStyle:{
fontSize:30
}
},
tooltip:{
trigger: 'item',
formatter: "视频:{b} 分享:{c}",
textStyle:{
fontSize:40
}
},
series:[
{
min:50,
max:200,
minSize: '20%',
type:"funnel",
data: data,
}
],
}
loudouEcharts.setOption(dataOption)
}
})
</script>
<span class="top-left border-span"></span>
<span class="top-right border-span"></span>
<span class="bottom-left border-span"></span>
<span class="bottom-right border-span"></span>
</div>
<div class="border-container">
<ul class="teacher-pie clearfix">
<div id="barq5"></div>
<script>
var rankEcharts = echarts.init(document.getElementById("barq5"))
var startIndex = 0
var endIndex =10
var timerId = 0
var allData = []
rankEcharts.on("mouseover",function(){
clearInterval(timerId)
})
rankEcharts.on("mouseout",function(){
startRankInterval()
})
$.ajax({
type:"GET",
url:"/table04",
dataType:"json",
success:function(data){
allData = data
province_arr = data.map(function(item){
return item.day;
})
value_arr = data.map(function(item){
return item.num
})
const option = {
title:{
text: '热门视频每日投币量Top5',
left:20,
top:20,
textStyle:{
fontSize:40,
color:"#b300ff"
},
},
grid:{
top:"25%",
left:"5%",
right:"5%",
bottom:"5%",
containLabel:true
},
dataZoom:{
show:false,
startValue:startIndex,
endValue:endIndex
},
tooltip:{
show:true,
textStyle:{
fontSize:30,
color:"#00ffff",
},
formatter: "日期:{b} 投币数:{c}"
},
xAxis:{
type:"category",
data:province_arr,
axisLabel: {
textStyle: {
color: "#00ffff"
},
fontSize:25
},
},
yAxis:{
type:"value",
axisLabel: {
textStyle: {
color: '#00ffff'
},
fontSize:25//调整坐标轴字体大小
}
},
series:[
{
type:"bar",
data:value_arr,
barWidth:66,
itemStyle:{
barBorderRadius:[33,33,0,0], //左上，右上，右下，左下
color:new echarts.graphic.LinearGradient(0,0,1,0,[
//颜色过渡的过渡值
{
//起始的颜色
offset:0,
color:'#00BFFF'
},
{
//100%位置的颜色
offset:1,
color:'#D8BFD8'
}
]),
}
}
]
}
rankEcharts.setOption(option)
startRankInterval()
}
})
function startRankInterval(){
if (timerId){
clearInterval(timerId)
}
timerId = setInterval(function(){
startIndex += 1
endIndex += 1
if (endIndex > allData.length -1 ){
startIndex = 0
endIndex = 10
}
const updateOption = {
dataZoom:{
startValue:startIndex,
endValue:endIndex
}
}
rankEcharts.setOption(updateOption)
},1000)
}
</script>
<span class="top-left border-span"></span>
<span class="top-right border-span"></span>
<span class="bottom-left border-span"></span>
<span class="bottom-right border-span"></span>
</div>
</div>
<div class="main-right">
<div class="border-container">
<div id="line"></div>
<script>
var line = echarts.init(document.getElementById("line"))
$.ajax({
type:"GET",
url:"/table03",
dataType:"json",
success:function(data){
nbData(data)
}
})
function nbData(data){
var lineValues = []
var lineNames = []
for(var i =0;i< data.length;i++){
lineValues.push(data[i].num)
lineNames.push(data[i].day)
}
const options = {
title:{
text:"每日热门视频播放量",
left:0,
top:20,
textStyle:{
fontSize:30,
color:"#b300ff"
},
},
tooltip:{
trigger:'axis',
axisPointer:{
lineStyle:{
color:'#00ffff',
}
}
,textStyle:{
fontSize:30,
color:"#00ffff",
},
formatter: "月份:{b} 人数:{c}"
},
//调整坐标系的位置
grid:{
top:'20%',
left:'3%',
right:"6%",
bottom: "3%",
containLabel:true
},
xAxis:{
type:"category",
data:lineNames,
axisLabel: {
interval:0,
// rotate:-40,
// //inside: false,//本来是true，改为false后将图形中的字移到图形外边
textStyle: {
color: "#00ffff"
},
fontSize:25//调整坐标轴字体大小
},
},
yAxis:{
// min:5.934,
// max:5.986,
type:"value",
axisLabel: {
textStyle: {
color: '#00ffff'
},
fontSize:25//调整坐标轴字体大小
}
},
textStyle: {
color: '#51ffe2',
fontSize :20
},
series:[
{
type:"line",
data:lineValues,
smooth: false,
lineStyle: {
// color: "#921aff",
width: 15,
},
itemStyle:{
barBorderRadius:[33,33,0,0], //左上，右上，右下，左下
color:new echarts.graphic.LinearGradient(0,0,1,0,[
//颜色过渡的过渡值
{
//起始的颜色
offset:0,
color:'#00BFFF'
},
{
//100%位置的颜色
offset:1,
color:'#D8BFD8'
}
]),
},
markPoint: {
data: [
{ type: 'max', name: 'Max' },
{ type: 'min', name: 'Min' }
],
symbol:'image://http://img30.360buyimg.com/poprx/jfs/t20023/329/2344745722/12670/a004c21d/5b3c2794Ndd1983e8.png',
symbolSize:[100, 70],
symbolOffset:[10,-50],//位置偏移
label: {
textStyle:{
fontSize:20,
color:"#0051ff"
},}
}
}
]
}
line.setOption(options)
}
</script>
<span class="top-left border-span"></span>
<span class="top-right border-span"></span>
<span class="bottom-left border-span"></span>
<span class="bottom-right border-span"></span>
</div>
<div class="border-container">
<div id="qi"></div>
<script>
var wd2 = echarts.init(document.getElementById("qi"))
$.ajax({
type:"GET",
url:"/table05",
dataType:"json",
success:function(data){
showsportnumlData(data)
}
})
function showsportnumlData(data){
const worldcound = data.map(function(item){
return {
name:item.up_name,
value:item.up_fans,
}
})
var option = {
title:{
text:"粉丝最多的up主Top10",
left:0,
top:20,
textStyle:{
fontSize:30,
color:"#b300ff",
}
},
tooltip:{
show:true
,textStyle:{
fontSize:30,
color:"#00ffff",
},
formatter: "UP:{b} 粉丝量:{c}"
},
series: [{
type: 'wordCloud',
sizeRange: [70, 135],
rotationRange: [-45, 0, 45, 90],
rotationStep: 45,
gridSize: 25 ,
shape: 'triangle-forward',
width: '100%',
height: '100%',
textStyle: {
normal: {
color: function () {
return 'rgb(' + [
Math.round(Math.random() * 400),
Math.round(Math.random() * 400),
Math.round(Math.random() * 400)
].join(',') + ')';
}
}
},
data:worldcound
}]
}
wd2.setOption(option)
}
function screenAdapter(){
const titleFontSize = document.getElementById("worldcound").offsetWidth / 100 * 3.6
const adapterOptions = {
title:{
textStyle:{
fontSize:titleFontSize
}
},
}
wd2.setOption(adapterOptions)
wd2.resize()
}
</script>
<span class="top-left border-span"></span>
<span class="top-right border-span"></span>
<span class="bottom-left border-span"></span>
<span class="bottom-right border-span"></span>
</div>
</div>
</div>
</body>
</html>
扫一扫
283
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
