利用WebClient和WebRequest类获得网页

最新推荐文章于 2022-05-23 17:52:24 发布

转载最新推荐文章于 2022-05-23 17:52:24 发布 · 708 阅读

文章标签：

#textbox #button #server #object #stream #microsoft

本文介绍了一种使用ASP.NET实现的简单方法来获取指定URL的网页源代码。通过两种方式实现：利用WebClient类和WebRequest类。同时展示了如何解析并显示这些内容。

GetPageHtml.aspx

&lt;%@ Page language=&quot;c#&quot; validateRequest = &quot;false&quot; Codebehind=&quot;GetPageHtml.aspx.cs&quot;
AutoEventWireup=&quot;false&quot; Inherits=&quot;eMeng.Exam.GetPageHtml&quot; %&gt;
&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0 Transitional//EN&quot; &gt;
&lt;HTML&gt;
&lt;HEAD&gt;
&lt;title&gt;得到网页源代码&lt;/title&gt;
&lt;meta name=&quot;GENERATOR&quot; Content=&quot;Microsoft Visual Studio 7.0&quot;&gt;
&lt;meta name=&quot;CODE_LANGUAGE&quot; Content=&quot;C#&quot;&gt;
&lt;meta name=&quot;vs_defaultClientScript&quot; content=&quot;JavaScript&quot;&gt;
&lt;meta name=&quot;vs_targetSchema&quot; content=&quot;http://schemas.microsoft.com/intellisense/ie5&quot;&gt;
&lt;/HEAD&gt;
&lt;body MS_POSITIONING=&quot;GridLayout&quot;&gt;
&lt;form id=&quot;aspNetBuffer&quot; method=&quot;post&quot; runat=&quot;server&quot;&gt;
&lt;div align=&quot;center&quot; style=&quot;FONT-WEIGHT: bold&quot;&gt;得到任意网页源代码&lt;/div&gt;
&lt;asp:TextBox id=&quot;UrlText&quot; runat=&quot;server&quot; Width=&quot;400px&quot;&gt;http://dotnet.aspx.cc/content.aspx
&lt;/asp:TextBox&gt;
&lt;asp:Button id=&quot;WebClientButton&quot; Runat=&quot;server&quot; Text=&quot;用WebClient得到&quot;&gt;&lt;/asp:Button&gt;
&lt;asp:Button id=&quot;WebRequestButton&quot; runat=&quot;server&quot; Text=&quot;用WebRequest得到&quot;&gt;&lt;/asp:Button&gt;
&lt;br&gt;
&lt;asp:TextBox id=&quot;ContentHtml&quot; runat=&quot;server&quot; Width=&quot;100%&quot; Height=&quot;360px&quot; TextMode=&quot;MultiLine&quot;&gt;
&lt;/asp:TextBox&gt;
&lt;/form&gt;
&lt;/body&gt;
&lt;/HTML&gt;

GetPageHtml.aspx.cs

using System;
using System.Collections;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Web;
using System.Web.SessionState;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Web.UI.HtmlControls;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
namespace eMeng.Exam
{
/// &lt;summary&gt;
/// GetPageHtml 的摘要说明。
/// &lt;/summary&gt;
public class GetPageHtml : System.Web.UI.Page
{
protected System.Web.UI.WebControls.Button WebClientButton;
protected System.Web.UI.WebControls.Button WebRequestButton;
protected System.Web.UI.WebControls.TextBox ContentHtml;
protected System.Web.UI.WebControls.TextBox UrlText;
protected System.Web.UI.WebControls.Button GetText;
private string PageUrl = &quot;&quot;;
private void Page_Load(object sender, System.EventArgs e)
{}
#region Web Form Designer generated code
override protected void OnInit(EventArgs e)
{
InitializeComponent();
base.OnInit(e);
}
/// &lt;summary&gt;
/// 设计器支持所需的方法 - 不要使用代码编辑器修改
/// 此方法的内容。
/// &lt;/summary&gt;
private void InitializeComponent()
{
this.WebClientButton.Click += new System.EventHandler(this.WebClientButton_Click);
this.WebRequestButton.Click += new System.EventHandler(this.WebRequestButton_Click);
this.GetText.Click += new System.EventHandler(this.GetText_Click);
this.Load += new System.EventHandler(this.Page_Load);
}
#endregion
private void WebClientButton_Click(object sender, System.EventArgs e)
{
PageUrl = UrlText.Text;
WebClient wc = new WebClient();
wc.Credentials = CredentialCache.DefaultCredentials;
///方法一：
Byte[] pageData = wc.DownloadData(PageUrl);
ContentHtml.Text = Encoding.Default.GetString(pageData);
/// 方法二：
/// ***************代码开始**********
/// Stream resStream = wc.OpenRead(PageUrl);
/// StreamReader sr = new StreamReader(resStream,System.Text.Encoding.Default);
/// ContentHtml.Text = sr.ReadToEnd();
/// resStream.Close();
/// **************代码结束********
///
wc.Dispose();
}
private void WebRequestButton_Click(object sender, System.EventArgs e)
{
PageUrl = UrlText.Text;
WebRequest  request = WebRequest.Create(PageUrl);
WebResponse response = request.GetResponse();
Stream resStream = response.GetResponseStream();
StreamReader sr = new StreamReader(resStream, System.Text.Encoding.Default);
ContentHtml.Text = sr.ReadToEnd();
resStream.Close();
sr.Close();
}
private void GetText_Click(object sender, System.EventArgs e)
{
PageUrl = UrlText.Text;
WebRequest  request = WebRequest.Create(PageUrl);
WebResponse response = request.GetResponse();
Stream resStream = response.GetResponseStream();
StreamReader sr = new StreamReader(resStream, System.Text.Encoding.Default);
ContentHtml.Text = sr.ReadToEnd();
resStream.Close();
sr.Close();
ContentHtml.Text = Regex.Replace(ContentHtml.Text,&quot;&lt;[^&gt;]*&gt;&quot;, &quot;&quot;);
//替换空格
ContentHtml.Text = Regex.Replace(ContentHtml.Text,&quot;//s+&quot;, &quot; &quot;);
}
}
}

某些网站的修正代码
string PageUrl = "http://www.blogcn.com/rss2/06sg.xml";
System.Net.HttpWebRequest request = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(PageUrl);
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1) Web-Sniffer/1.0.24";
System.Net.WebResponse response = request.GetResponse();
System.IO.Stream resStream = response.GetResponseStream();
System.IO.StreamReader sr = new System.IO.StreamReader(resStream, System.Text.Encoding.Default);
ContentHtml.Text = sr.ReadToEnd();
resStream.Close();
sr.Close();

关于协议冲突：在ASP.NET 2.0里使用web.config设置：

<system.net>
<settings>
<httpWebRequest useUnsafeHeaderParsing = "true"/>
</settings>
</system.net>