Koa2 + Puppeteer打造『爬虫系统』7

最新推荐文章于 2024-08-18 21:00:21 发布

原创最新推荐文章于 2024-08-18 21:00:21 发布 · 289 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#node.js #爬虫 #javascript

实战项目专栏收录该内容

24 篇文章

订阅专栏

本文介绍如何创建课程和关于我们页面的数据模型，并实现数据的抓取及存储过程。包括使用Node.js进行数据模型定义、爬虫开发、以及数据同步到数据库的具体步骤。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

十九.创建课程数据模型以及数据入表操作

1.创建course.js的模型

const seq = require('../connection/mysql_connect'),
   { STRING,INT } = require('../../config/db_type_config');
const Course = seq.define('course',{
    cid:{
        comment: 'Course ID',
        type:INT,
        allowNull:false,
        unique:true
    },
    href:{
        comment: 'Course detail page link',
        type:STRING,
        allowNull:false
    },
    posterUrl:{
        comment:'Course img url',
        type:STRING,
        allowNull:false
    },
    courseName:{
        comment:'Course name',
        type:STRING,
        allowNull:false
    },
    price:{
        comment:'Course price',
        type:STRING,
        allowNull:false
    },
    description:{
        comment:'Course description',
        type:STRING,
        allowNull:false
    },
    studentCount:{
        comment:'Course count',
        type:INT,
        allowNull:false
    },
    field:{
        comment:'the course the tab',
        type:INT,
        allowNull:false
    },
    posterKey:{
        comment:'qiniu course image name',
        type:STRING,
        allowNull:false
    },
    status:{
        comment:'course status',
        type:INT,
        defaultValue:1,
        allowNull:false
    }

    
})   
module.exports = Course;

2.入口导入

const Slider = require('./slider'),
      RecomCourse = require('./recomCourse'),
      AgencyInfo = require('./agencyinfo'),
      Collection = require('./collection'),
      Teacher =  require('./teacher'),
      Student = require('./student'),
      CourseTab = require('./courseTab')
      Course = require('./course')

module.exports={
    Slider,AgencyInfo,RecomCourse,Collection,Teacher,Student,CourseTab,Course
}

node do/sync.js同步表

3.在service中写入course.js

const CourseModel = require('../do/models/course');
class CourseService{
    async addCourse(data){
        const cid = data.cid;
        const result = await CourseModel.findOne({
            where:{cid}
        })
        if(result){
            return await CourseModel.update(data,{
                where:{cid}
            })
        }else{
            return await CourseModel.create(data)
        }
    }
}
module.exports = new CourseService();

4.完善控制器

 {addCourse} = require('../service/course')
const result = await addCourse(item);
                       if(result){
                           console.log('Data create Ok')
                       }else{
                           console.log('Data create failed')
                       }

二十.爬取关于我们数据、建立表模型、数据入表

1.配置config.js

module.exports={
    qiniu:{
        keys:{
            ak:'qMK7okFyL1xX2o8gjFzy1PrI_jXR_yf58naIcIs8',
            sk:'j6Ngxxy6xSzXi9czKXun0Uju_5hDahAla9THml9C'
        },
        bucket:{
            tximg:{
                bucket_name:'crawler-txclass',
                domain:'http://qexgb3yqx.hn-bkt.clouddn.com'
            }
        }
    },
    crawler:{
        url:{
            main:'https://msiwei.ke.qq.com/#tab=0&category=-1',
            course:'https://msiwei.ke.qq.com/#tab=1&category=-1',
            teacher:'https://msiwei.ke.qq.com/#tab=2&category=-1',
            aboutus:'https://msiwei.ke.qq.com/#category=-1&tab=3'
        }
    }
}

2.建立aboutus.js爬虫文件

const Crawler = require('../lib/crawler'),
      {crawler} = require('../config/config');
Crawler({
    url:crawler.url.aboutus,
    callback(){
        const $ = window.$,
              $wrapper = $('.agency-about');
        return {
            aid:1,
            posterUrl:$wrapper.find('.about-banner-pic0').css('background-image').match(/\"(.+?)\"/)[1],
            title:$wrapper.find('.about-agency-propagate').text(),
            name:$wrapper.find('.about-agency-name').text(),
            intro:$wrapper.find('.about-agency-intr').text(),
            posterKey:''
        }      
    }
})

3.编辑路由

router.get('/crawl_aboutus',crawlerController.crawlAboutus )

4.编写控制器

crawlAboutus(){
        startProcess({
            path:'../crawler/aboutus',
            async message(data){
                if(data.posterUrl && !data.posterKey){
                    try {
                        const posterData = await qiniuUpload({
                            url:data.posterUrl,
                            bucket:qiniu.bucket.tximg.bucket_name,
                            ext:'.jpg'
                        })
                        if(posterData.key){
                            data.posterKey = posterData.key
                        }
                    } catch (error) {
                        console.log(error)
                    }
                }
            },
            async exit(data){
                console.log(data);
            },
            async error(data){
                console.log(data);
            }
        })
    }

5.建立表模型

const seq = require('../connection/mysql_connect'),
   { STRING,INT ,TEXT} = require('../../config/db_type_config');
const Aboutus = seq.define('aboutus',{
    aid:{
        comment:'aboutus Id',
        type:INT,
        allowNull:false,
        unique:true
    },
    posterUrl:{
        comment:'poster',
        type:STRING,
        allowNull:false
    },
    title:{
        comment:'title',
        type:STRING,
        allowNull:false
    },
    name:{
        comment:'NAME',
        type:STRING,
        allowNull:false
    },
    intro:{
        comment:'introduction',
        type:TEXT,
        allowNull:false
    },
   
    posterKey:{
        comment:'qiniu poster image name',
        type:STRING,
        allowNull:false
    }
})   
module.exports = Aboutus;

6.入口文件导入

const Slider = require('./slider'),
      RecomCourse = require('./recomCourse'),
      AgencyInfo = require('./agencyinfo'),
      Collection = require('./collection'),
      Teacher =  require('./teacher'),
      Student = require('./student'),
      CourseTab = require('./courseTab')
      Course = require('./course'),
      Aboutus =  require('./aboutus'),

module.exports={
    Slider,AgencyInfo,RecomCourse,Collection,Teacher,Student,CourseTab,Course,Aboutus
}

7.建立Aboutus.js在service中

const AboutusModel = require('../do/models/aboutus');
class AboutusService {
    async addAboutus(data){
        const id = data.aid;
        const result = await AboutusModel.findOne({
            where:{id}
        });
        if(result){
            return await AboutusModel.update(data,{
                where:{id}
            })
        }else{
            return await AboutusModel.create(data);
        }
    }
}
module.exports=new AboutusService();

8.控制器完善

 const result = await addAboutus(item);
                        if(result){
                            console.log('Data create Ok')
                        }else{
                            console.log('Data create failed')
                        }