拉钩网采集

<?php
  //var_dump($_SERVER);exit;
  $cookie_file = tempnam('./temp','cookie');  //创建cookie文件保存的位置
  //echo $cookie_file;exit;
 function  curl($url,$data=array(),$method,$setcooke=false,$cookie_file=false){
        $ch = curl_init();     //1.初始化
        curl_setopt($ch, CURLOPT_URL, $url); //2.请求地址
        curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method);//3.请求方式
        //4.参数如下    禁止服务器端的验证
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
        curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, FALSE);
        //伪装请求来源，绕过防盗
       //curl_setopt($ch,CURLOPT_REFERER,"http://wthrcdn.etouch.cn/");
       //配置curl解压缩方式（默认的压缩方式）
       curl_setopt($ch, CURLOPT_HTTPHEADER, array('Accept-Encoding:gzip'));
       curl_setopt($ch, CURLOPT_ENCODING, "gzip");

        curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:44.0) Gecko/20100101 Firefox/44.0'); //指明以哪种方式进行访问
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
        curl_setopt($ch, CURLOPT_AUTOREFERER, 1);
        if($method=="POST"){//5.post方式的时候添加数据    
            curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
        }
        if($setcooke==true){
             //如果设置要请求的cookie，那么把cookie值保存在指定的文件中
            curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_file);
        }else{
             //就从文件中读取cookie的信息
            curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie_file);
        }
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        $tmpInfo = curl_exec($ch);

        if (curl_errno($ch)) {
            return curl_error($ch);
        }
        curl_close($ch);
        return $tmpInfo;
    }
    /*
  //模拟get请求
   $url="https://www.taobao.com/";
   $str=curl($url,array(),'GET');
   //echo $str;
  //post请求
   $url="http://www.study.com/seven7/1408phpB/20160311/1.php";
   $data=array('username'=>'abc');
   $str=curl($url,$data,'POST');
   echo $str;
   */
   //模拟登陆
    //登陆保存用户信息到cookie中
    header("content-type:text/html;charset=utf-8");
    set_time_limit(0);
    $url="http://www.lagou.com/jobs/positionAjax.json?city=%E5%8C%97%E4%BA%AC&needAddtionalResult=false";
    $data=array('first'=>'false','kd'=>'php','pn'=>2);
    //post提交
    $str = curl($url,$data,'POST');
    $arr = json_decode($str,true);
    // var_dump($arr);die;
    // echo $str;die;
    $arr=$arr['content']['positionResult']['result'];
    // var_dump($arr);die;
    $pdo = new PDO("mysql:host=127.0.0.1;dbname=demo",'root','root');
    $pdo->exec("set names utf8");
    foreach ($arr as $key => $val) {
        $data1 = $val['companyShortName'];
        $data2 = $val['positionName'];
        $data3 = $val['district'];
        $data4 = $val['workYear'];
        $data5 = $val['education'];
        $sql = "insert into lagou (companyShortName,positionName,district,workYear,education) values('$data1','$data2','$data3','$data4','$data5')";
        $pdo->exec($sql);
    }

    //get获取
    // $url="http://www.ecshop.com/user.php?act=order_list";
    // $str=curl($url,array(),'GET',false,$cookie_file);
    // echo $str;