nodejs 采集新闻数据

var express = require( 'express' );

var $ = require( 'jQuery' );

var app = express();

var colors = require( 'colors' );

var message_list = require( './zui/message_list' );

var findData = require( './findData' );

//设置全局跨域访问 

app.all( '*' , function(req, res, next) { 

     res.header( "Access-Control-Allow-Origin" , "*" ); 

     res.header( "Access-Control-Allow-Headers" , "Origin, X-Requested-With, Content-Type, Accept" ); 

     res.header( "Access-Control-Allow-Methods" , "PUT,POST,GET,DELETE,OPTIONS" ); 

     res.header( "X-Powered-By" , ' 3.2.1' ) 

     res.header( "Content-Type" , "application/json;charset=utf-8" );

     next(); 

});

//输出主页列表数据

app.get( '/getIndexData' , function (req, res) {

     findData.indexData(req, res);

});

/*

*输出列表数据。支持范围选择

*demo：http://localhost:3000/getList/0,9 前9条数据 从0开始

*demo: http://localhost:3000/getList/0  所有的数据  从0开始

*/

app.get(/^\/getList?(?:\/(\d+)(?:\,(\d+))?)?/, function (req, res) {

     var start=undefined,end=undefined;

     if (req.params.length== 1 ){

         start=req.params[ 0 ];

     } else if (req.params.length== 2 ){

         start=req.params[ 0 ];

         end=req.params[ 1 ];

     }

     findData.list(req, res,start,end,function(data){

         for (var i= 0 ,j=data.length;i<j;i++){ data[i].orderby= "1" ;= "" }= "" return = "" data;= "" });= "" 数据列表= "" var= "" newslistdata= "undefined;" app.get( '="" getnewslist' ,= "" function= "" (req,= "" res)= "" {= "" if (typeof= "" finddata.list(req,= "" res,undefined,undefined,function(data,length){= "" for (var= "" i= "0,j=data.length;i<j;i++){" data[i].summary= "暂无摘要" data[i].updateusername= "admin" getnewslist(req,= "" res);= "" } else {= "" 缓存数据列表= "" res){= "" pagenum= "req.query.pageNum,pageSize=req.query.pageSize;" start= "(pageNum-1)*pageSize,end=pageNum*pageSize;" newdata= "newsListData.slice(start," end);= "" res.send(json.stringify(newdata));= "" app.listen( 3000 );= "" console.log( "nodejs=" " was=" " start" .green);= "" <= "" pre= "" ><br>

<pre class = "brush:java;" >var http = require( 'http' ),

     fs = require( 'fs' ),

     jquery = fs.readFileSync( "lib/jquery.min.js" , "utf-8" ),

     jsdom = require( 'jsdom' );

function get(url, callback) {

     //使用代理

         var opt = {

         host: '127.0.0.1' ,

         port: '7070' ,

         method: 'get' , //这里是发送的方法

         path: url

     }

     //以下是接受数据的代码

     var req = http.request(opt, function(res) {

         res.setEncoding( 'utf8' );

         var html = '' ;

         res.on( 'data' , function(d) {

             html += d;

         }).on( 'end' , function() {

             jsdom.env({

                 html: html,

                 src: [jquery],

                 done: function(errors, window) {

                     var $ = window.$;

                     callback && callback(errors, $);

                     window.close(); // 释放window相关资源，否则将会占用很高的内存

                 }

             });

         });

     }).on( 'error' , function(e) {

         console.log( "Got error: " + e.message);

     })

     req.end();

}

//导出list

exports.indexData = function(request, response, start, end) {

     get( 'http://www.news.cn/edu/index.htm' , function(errors, $) {

         var $list = $( ".list" );

         var data = [];

         $list.each(function(index) {

             var $ this = $( this ),

                 $li = $ this .find( "li" ),

                 li_data = [];

             $li.each(function(index) {

                 var $a = $( this ).find( "a" );

                 var href=$a.attr( "href" ).replace( "http://news.xinhuanet.com/edu/" , "" );

                 var temp_href= "/" +href.replace(/[^c]*/, "" );

                 var date=href.replace(temp_href, "" ).replace( "/" , "-" );

                 if (date.length> 10 ){

                     date= "2013-01-04" ;

                 }

                 li_data.push({

                     title: $a.text(),

                     href: $a.attr( "href" ),

                     date:date,

                     id: index

                 });

             });

             data.push({

                 id: index,

                 list: li_data

             });

         });

         if (typeof start != "undefined" && typeof end == "undefined" ) {

             data = data.slice(start);

         } else if (typeof start != "undefined" && typeof end != "undefined" ) {

             data = data.slice(start, end);

         }

         response.send(JSON.stringify(data));

     });

}

//导出list

exports.list = function(request, response, start, end,callback) {

     get( 'http://www.news.cn/edu/index.htm' , function(errors, $) {

         var $li = $( ".list li" );

         var data = [];

         $li.each(function(index) {

             var $a = $( this ).find( "a" );

             var href=$a.attr( "href" ).replace( "http://news.xinhuanet.com/edu/" , "" );

             var temp_href= "/" +href.replace(/[^c]*/, "" );

             var date=href.replace(temp_href, "" ).replace( "/" , "-" );

             if (date.length> 10 ){

                 date= "2013-01-04" ;

             }

             

             data.push({

                 id: index,

                 title: $a.text(),

                 attachment: $a.attr( "href" ),

                 updateTime:date

             });

         });

         var newData=data;

         if (typeof start != "undefined" && typeof end == "undefined" ) {

             newData = data.slice(start);

         } else if (typeof start != "undefined" && typeof end != "undefined" ) {

             newData = data.slice(start, end);

         }

         if (typeof callback== "function" ){

             newData=callback(newData,data.length);

         }

         //response.header("Access-Control-Allow-Origin", "*");   //设置跨域访问 

         response.send(JSON.stringify(newData));

     });

}</pre><br>

访问地址<p></p>

<p></p><pre class = "brush:java;" >http: //localhost:3000/getIndexData

http: //localhost:3000/getList/0,4

http: //localhost:3000/getNewsList?pageNum=1&pageSize=10</pre><p></p>

<p>over<br>

</p>                        </j;i++){>

​​​​​​​

延伸阅读

---

▶ Walkthrough007