twitter crawler POST GET

 

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;

namespace Twitter_Login_Crawler
{
    class Program
    {
        static void Main(string[] args)
        {
            Html_Dowmload();
        }
        static void Html_Dowmload()
        {
            //
            var homeUrl = "https://twitter.com/";
            var encoding = "utf-8";
            //
            string authenticity_token;
            //
            WebProxy proxy = new WebProxy();
            proxy.Address = new Uri("http://127.0.0.1:8087");
            //GET请求
            HttpWebRequest request = null;
            HttpWebResponse response = null;
            CookieContainer _cc = new CookieContainer();
            string _cookies = string.Empty;
            Encoding _encoding = Encoding.GetEncoding(encoding);
            request = (HttpWebRequest)WebRequest.Create(homeUrl);
            request.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:14.0) Gecko/20100101 Firefox/14.0.1";
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
            request.ContentType = "application/x-www-form-urlencoded";
            request.AllowAutoRedirect = true;
            request.CookieContainer = _cc;
            request.KeepAlive = true;
            request.Method = "GET";
            request.KeepAlive = true;
            request.Headers.Add("Cookie:" + _cookies);
            //
            request.Proxy = proxy;
            try
            {
                //接收响应
                response = (HttpWebResponse)request.GetResponse();
                //设置cookie   
                _cookies = request.CookieContainer.GetCookieHeader(request.RequestUri);
                //取再次跳转链接   
                var sr = new StreamReader(response.GetResponseStream(), _encoding);
                var currentPage = sr.ReadToEnd();
                Regex re = new Regex(@"name=""authenticity_token"" value=""(.+?)""", RegexOptions.None);
                Match mc = re.Match(currentPage);
                authenticity_token = mc.Groups[1].Value;
            }
            catch
            {
                throw new Exception("1.get twitter page error!");
            }
            //POST请求  
            string postData = string.Format(@"session%5Busername&session%5Bpaity_token={0}", authenticity_token);
            string loginUrl = "https://twitter.com/sessions";
            request = (HttpWebRequest)WebRequest.Create(loginUrl);
            request.Method = "POST";
            request.ContentType = "application/x-www-form-urlencoded";
            byte[] postdatabytes = Encoding.UTF8.GetBytes(postData);
            request.ContentLength = postdatabytes.Length;
            request.Referer = "	https://twitter.com/";
            request.AllowAutoRedirect = false;
            request.CookieContainer = _cc;
            request.KeepAlive = true;
            request.Headers.Add("Cookie:" + _cookies);
            request.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:14.0) Gecko/20100101 Firefox/14.0.1";
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
            request.ContentType = "application/x-www-form-urlencoded";
            //
            request.Proxy = proxy;
            try
            {
                Stream stream;
                stream = request.GetRequestStream();
                stream.Write(postdatabytes, 0, postdatabytes.Length);
                stream.Close();
                //接收响应
                response = (HttpWebResponse)request.GetResponse();
                //设置cookie   
                _cookies = request.CookieContainer.GetCookieHeader(request.RequestUri);
                var sr = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
                var currentPage = sr.ReadToEnd();
            }
            catch 
            {
                throw new Exception("2.post twitter page error!");
            }
            //GET请求
            var searchUrl = "https://twitter.com/search?q=test&src=typd";
            request = (HttpWebRequest)WebRequest.Create(searchUrl);
            request.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:14.0) Gecko/20100101 Firefox/14.0.1";
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
            request.ContentType = "application/x-www-form-urlencoded";
            request.AllowAutoRedirect = true;
            request.CookieContainer = _cc;
            request.KeepAlive = true;
            request.Method = "GET";
            request.KeepAlive = true;
            request.Headers.Add("Cookie:" + _cookies);
            //
            request.Proxy = proxy;
            try
            {
                //接收响应
                response = (HttpWebResponse)request.GetResponse();
                //设置cookie   
                _cookies = request.CookieContainer.GetCookieHeader(request.RequestUri);
                //取再次跳转链接   
                var sr = new StreamReader(response.GetResponseStream(), _encoding);
                var currentPage = sr.ReadToEnd();
            }
            catch
            {
                throw new Exception("3.get twitter page error!");
            }
        }
    }
}