C# HttpWebRequest和HttpWebRespons笔记

private string GetHTML(string url, string cookie) { if (String.IsNullOrEmpty(url)) { return null; } else { try { url = url.Replace(" ", "%20"); url = url.Replace("&", "&"); } catch { } } string html = null; string curCookie = null; HttpWebRequest myRequest = null; HttpWebResponse myResp = null; StreamReader myReader = null; for (int i = 0; i < 3; i++) { try { myRequest = (HttpWebRequest)WebRequest.Create(url); myRequest.AllowAutoRedirect = false; myRequest.Timeout = 60 * 1000; myRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"; myRequest.Headers.Add("Cookie", cookie); myRequest.KeepAlive = true; myResp = (HttpWebResponse)myRequest.GetResponse(); curCookie = myResp.Headers["Set-Cookie"]; curCookie = curCookie.Substring(0, curCookie.IndexOf(";")); myReader = new StreamReader(myResp.GetResponseStream(), Encoding.Default, true); html = myReader.ReadToEnd(); } catch (Exception e) { // Console.WriteLine(e.ToString()); continue; } finally { if (myReader != null) { myReader.Close(); } if (myResp != null) { myResp.Close(); } } if (html == null) { continue; } else if (html.IndexOf("Object moved") > 0) //手动获取转向地址,再次获取html代码 { int pos1 = html.IndexOf("http://"); int pos2 = html.IndexOf("/">"); url = html.Substring(pos1, pos2 - pos1); html = GetHTML(url, curCookie); } break; } return html; } 

//获取指定页面的源代码 public String GetPageCode(String PageURL, String Charset) { try { //存放目标网页的html String strHtml = ""; //连接到目标网页 HttpWebRequest wreq = (HttpWebRequest)WebRequest.Create(PageURL); //wreq.Headers.Add("VIA", "gotest"); //wreq.Headers.Add(HttpRequestHeader.UserAgent, "ddd"); // wreq.Headers.Add(HttpRequestHeader.UserAgent, "Mozilla/4.0 (compatible; MSIE 6.0; " + "Windows NT 5.2; .NET CLR {0}"); //wreq.Headers.Add(HttpResponseHeader.Upgrade, "MozillaTTTT"); //wreq.Headers.Add(HttpRequestHeader.UserAgent, "Mozilla"); //wreq.Headers.Add("User-Agent", "winnnnnnnnnnnnn"); //wreq.Headers.Add("X_FORWARDED_FOR", "10.0.0.1"); //发送X_FORWARDED_FOR头(若是用取源IP的方式，可以用这个来造假IP,对日志的记录无效) //wreq.Headers.Add(HttpRequestHeader.UserAgent, "MIME类型"); //wreq.Headers.Add("HTTP_USER_AGENT","111"); //wreq.Timeout = 30000; //设置连接超时时间 //wreq.Headers.Set("Pragma", "no-cache"); //不要缓存 //wreq.Headers.Set(HttpRequestHeader.UserAgent, @"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322; Alexa Toolbar)"); //wreq.UserAgent = "From lvwp Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322; Alexa Toolbar)"; //wreq.Headers.Add("Translate: f"); //wreq.Headers.Set("HTTP_REFERER","www.tvsou.com"); //wreq.Headers.Add("HTTP_REFERER", "www.tvsou.com"); wreq.Referer = "http://tjtv.enorth.com.cn/jmsj/ws.htm"; //设置HTTP_REFERER wreq.Headers.Add("X_FORWARDED_FOR", "101.0.0.11"); //发送X_FORWARDED_FOR头(若是用取源IP的方式，可以用这个来造假IP,对日志的记录无效) wreq.Method = "Get"; wreq.KeepAlive = true; wreq.ContentType = "application/x-www-form-urlencoded"; wreq.AllowAutoRedirect = true; wreq.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; wreq.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)"; CookieContainer cookieCon = new CookieContainer(); wreq.CookieContainer = cookieCon; HttpWebResponse wresp = (HttpWebResponse)wreq.GetResponse(); //9tvtest //采用流读取，并确定编码方式 Stream s = wresp.GetResponseStream(); StreamReader objReader = new StreamReader(s, System.Text.Encoding.GetEncoding(Charset)); string strLine = ""; //读取 while (strLine != null) { strLine = objReader.ReadLine(); if (strLine != null) { strHtml += strLine.Trim(); } } strHtml = strHtml.Replace("<br />", "/r/n"); return strHtml; } catch (Exception n) //遇到错误，打印错误 { return n.Message; } }

 

注意：尝试自动重定项的次数太多的解决方案： HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
CookieContainer cc = new CookieContainer();
req.CookieContainer = cc;