asp.net 网页抓取

Html代码：

<form id="form1" runat="server"> <div> <table border="0" cellpadding="0" cellspacing="0" style="width: 100%"> <tr> <td style="width: 100%"> 网址 <asp:TextBox ID="TextBox1" runat="server" Width="383px"></asp:TextBox> <asp:Button ID="Button1" runat="server" Text="获取" OnClick="Button1_Click" Width="88px" /></td> </tr> <tr> <td style="width: 100%"> <asp:GridView ID="GridView1"　 Width="95%" runat="server" AutoGenerateColumns="False"> <Columns> <asp:HyperLinkField DataNavigateUrlFields="Url,Title" DataNavigateUrlFormatString="./HtmlFiles/{0}" DataTextField="Title" HeaderText="标题" /> <asp:BoundField DataField="BuildTime" HeaderText="创建时间" /> <asp:HyperLinkField DataNavigateUrlFields="LocalUrl" DataNavigateUrlFormatString="./HtmlFiles/{0}" DataTextField="LocalUrl" HeaderText="原始网址" /> </Columns> </asp:GridView> </td> </tr> </table> </div> </form>

 

C#代码：

protected void Page_Load(object sender, EventArgs e) { if (!Page.IsPostBack) { // 初始化一个内存表，用来存放已经抓取并生成好的静态页面的相关信息 DataTable dt = new DataTable(); DataColumn dc; dc = new DataColumn("Title", typeof(string)); dt.Columns.Add(dc); dc = new DataColumn("Url", typeof(string)); dt.Columns.Add(dc); dc = new DataColumn("BuildTime", typeof(DateTime)); dt.Columns.Add(dc); dc = new DataColumn("LocalUrl", typeof(string)); dt.Columns.Add(dc); Session["HtmlFiles"] = dt; } } protected void Button1_Click(object sender, EventArgs e) { string url = TextBox1.Text.Trim(); string content = ""; if (url.Length > 0) { WebRequest webRequest = WebRequest.Create(url); WebResponse webResponse = webRequest.GetResponse(); StreamReader sr = new StreamReader(webResponse.GetResponseStream(), System.Text.Encoding.GetEncoding("gb2312")); content = sr.ReadToEnd(); // 提取页面标题，即HMTL代码中<title>和</title>之间的文字 // 在这个方法，注意 StreamReader中使用的Encoding， //如果和页面的一致，可以正确获取，如果不一致，可能获取为乱码 int s = content.LastIndexOf("<title>") + 7; int l = content.LastIndexOf("</title>") + 8; string title = ""; if (s > 0 & l > 0 & l > s) title = content.Substring(s, l - s); else title = "UnKnown"; string htmlFileName = ""; if (content.Length > 0) { htmlFileName = DateTime.Now.ToString("yyyyMMdd-hhmmss") + ".html"; //将获取的网页保存为静态页面 System.IO.StreamWriter sw = new StreamWriter(Server.MapPath("./HtmlFiles/") + htmlFileName, false, System.Text.Encoding.GetEncoding("gb2312")); try { sw.Write(content); } catch { throw (new Exception("程序执行出错·！·")); return; } //将获取的页面信息 DataTable dt = (DataTable)Session["HtmlFiles"]; DataRow dr = dt.NewRow(); dr["Title"] = title; dr["Url"] = htmlFileName; dr["BuildTime"] = DateTime.Now.ToString(); dr["LocalUrl"] = url; // 保存至内存表中 // 实际应用中，我们将这些信息保存至数据库中 dt.Rows.Add(dr); //调用显示方法 GridViewBind(); string js = "<mce:script type="text/javascript"><!-- alert(/"生成成功！/"); // --></mce:script>"; this.Page.ClientScript.RegisterClientScriptBlock(this.GetType(), "Msg", js); sw.Flush(); sw.Close(); sw.Dispose(); } } } //将内存表中的内容利用GridView控件显示出来 private void GridViewBind() { DataTable dt = (DataTable)Session["HtmlFiles"]; this.GridView1.DataSource = dt; this.GridView1.DataBind(); }