自己写的采集

using System;
using System.Data;
using System.Configuration;
using System.Collections;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Web.UI.HtmlControls;

using System.IO;
using System.Net;
using System.Text;
using Dang.Utils;

namespace MyTest.CaiJi
...{
    public partial class caiji01 : System.Web.UI.Page
    ...{
        public static string reAllListHtml,reAllHtml,htmlAddress,flashAddress;
        public static string RelativeWay, RootWay, htmlListLu;
        public static ArrayList alist;//地址列表
        public static int getSavedNum;
        protected void Page_Load(object sender, EventArgs e)
        ...{
            getSavedNum = 0;
        }

        protected void Button1_Click(object sender, EventArgs e)
        ...{
            htmlAddress = TextBox1.Text.Trim().ToLower();

            RelativeWay = htmlAddress.Substring(0, htmlAddress.LastIndexOf("/")) + "/";//相对路径
            RootWay = htmlAddress.Substring(0, htmlAddress.IndexOf("/", 7)) + "/";//根路径
            
            reAllListHtml = GetPageHTML(htmlAddress);
            Panel1.Visible = true;
        }
        

        protected void Button3_Click(object sender, EventArgs e)
        ...{
            string starstr = TextBox2.Text.Trim();
            string endstr = TextBox3.Text.Trim();
            alist = Get_fileAddresss(starstr, endstr, reAllListHtml);
            Panel1.Visible = false;
            StringDo.Visible = true;
        }

        protected void Button2_Click(object sender, EventArgs e)
        ...{
            for (int i = 0; i < alist.Count; i++)
            ...{
                getflash(alist[i].ToString());
            }   
            
        }

        protected void Go_Click(object sender, EventArgs e)
        ...{
            for (int i = 0; i < alist.Count; i++)
            ...{
                getflash(alist[i].ToString());
            }   
        }


        public void getflash(string url)
        ...{
            reAllHtml = GetPageHTML(url);
            string starstr = StartUrlString.Text.Trim();
            string endstr = EndUrlString.Text.Trim();
            flashAddress = Get_fileAddress(starstr, endstr, reAllHtml);
            bool isSave = false;
            if (flashAddress != null)
            ...{
                flashAddress = Tohttp(flashAddress);
                isSave = SaveFileFromUrl(flashAddress);
            }           

            
            if (isSave)
                Label1.Text += url + "成功！<br />";
            else
                Label1.Text += url + "失败！<br />";
            getSavedNum += 1;
        }


        public string Tohttp(string str)
        ...{
            if (str.StartsWith("/"))
                str = RootWay + str;
            if (!str.StartsWith("http://"))
                str = RelativeWay + str;
            return str;
        }
        public string Get_fileAddress(string startstr,string endstr,string strResult)
        ...{
            String temp = "";
            int start, stop;

            start = strResult.IndexOf(startstr, 0, strResult.Length);
            stop = strResult.IndexOf(endstr, 0, strResult.Length);
            if (start == -1 || stop == -1)
                return null;
            temp = strResult.Substring(start+startstr.Length, stop-start-startstr.Length);
            return temp;
        }

        public ArrayList Get_fileAddresss(string startstr, string endstr, string strResult)
        ...{
            ArrayList list = new ArrayList();
            int start = 0, stop = 0;
            while (start != -1 && stop!=-1)
            ...{
                start = strResult.IndexOf(startstr, 0, strResult.Length);
                if (start == -1)
                    break;
                strResult = strResult.Substring(start + startstr.Length);
                stop = strResult.IndexOf(endstr, 0, strResult.Length);
                if ( stop == -1)
                    break;
                string tempaddress = strResult.Substring(0, stop);
                tempaddress = Tohttp(tempaddress);
                list.Add(tempaddress);
                strResult = strResult.Substring(stop);
            }            

            return list;
        }


        /**//// <summary>
        /// 从文件地址下载文件到本地磁盘
        /// </summary>
        /// <param name="Url">文件网址</param>
        /// <returns></returns>        
        public  bool SaveFileFromUrl(string Url)
        ...{
            if (Url.IndexOf(".") == -1)
                return false;
            string fileExt = Url.Substring(Url.LastIndexOf(".") + 1);
            bool Value = false;
            WebResponse response = null;
            Stream stream = null;

            try
            ...{
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);

                response = request.GetResponse();
                stream = response.GetResponseStream();

                if (!response.ContentType.ToLower().StartsWith("text/"))
                ...{
                    Value = SaveBinaryFile("flashsrc", fileExt, response);
                }

            }
            catch (Exception err)
            ...{
                string aa = err.ToString();
            }
            return Value;
        }
       
        /**//// <summary>
        ///  将二进制文件保存到磁盘
        /// </summary>
        /// <param name="fileDirectory">保存的目录flashsrc</param>
        /// <param name="fileNameExt">保存的类型</param>
        /// <param name="response">网络响应</param>
        /// <returns></returns>
        private  bool SaveBinaryFile(string fileDirectory,string fileNameExt,WebResponse response)
        ...{
            bool Value = true;
            byte[] buffer = new byte[1024];
            string dirpath = Server.MapPath("/"+fileDirectory+"/");
            if (Directory.Exists(dirpath) == false)
            ...{
                Directory.CreateDirectory(dirpath);
            }
            try
            ...{
                string FileName = dirpath + GetUniquelyString() + "." + fileNameExt;
                if (File.Exists(FileName))
                    File.Delete(FileName);
                Stream outStream = System.IO.File.Create(FileName);
                Stream inStream = response.GetResponseStream();

                int l;
                do
                ...{
                    l = inStream.Read(buffer, 0, buffer.Length);
                    if (l > 0)
                        outStream.Write(buffer, 0, l);
                }
                while (l > 0);

                outStream.Close();
                inStream.Close();
            }
            catch
            ...{
                Value = false;
            }
            return Value;
        }

        /**//// <summary>
        ///  获取一个不重复的文件名
        /// </summary>
        /// <returns></returns>
        public static string GetUniquelyString()
        ...{
            const int RANDOM_MAX_VALUE = 1000;
            string strTemp, strYear, strMonth, strDay, strHour, strMinute, strSecond, strMillisecond;
            Random rnd = new Random();
            DateTime dt = DateTime.Now;
            int rndNumber = rnd.Next(RANDOM_MAX_VALUE);
            strYear = YieldRandNum(5);
            strMonth = (dt.Month > 9) ? dt.Month.ToString() : "i" + dt.Month.ToString();
            strDay = (dt.Day > 9) ? dt.Day.ToString() : "a" + dt.Day.ToString();
            strHour = (dt.Hour > 9) ? dt.Hour.ToString() : "n" + dt.Hour.ToString();
            strMinute = (dt.Minute > 9) ? dt.Minute.ToString() : "j" + dt.Minute.ToString();
            strSecond = (dt.Second > 9) ? dt.Second.ToString() : "n" + dt.Second.ToString();
            strMillisecond = dt.Millisecond.ToString();

            strTemp = strYear + strDay + strMonth + strHour + strMinute + strSecond + strMillisecond + rndNumber.ToString();
            strTemp = strTemp.Replace("1", "q");

            return strTemp;

        }

        /**//// <summary>
        /// 产生随机字母
        /// </summary>
        /// <param name="d"></param>
        /// <returns></returns>
        public static string YieldRandNum(int d)
        ...{
            char[] seed =...{ 'A', 'b', 'B', 'c', 'C', 'd', 'D', 'e', 'E', 'f', 'F', 'G', 'h', 'H', 'i', 'j', 'J', 'k', 'K', 'L', 'm', 'M', 'n', 'N', 'p', 'P', 'q', 'Q', 'R', 's', 'S', 't', 'T', 'u', 'U', 'v', 'V', 'w', 'W', 'x', 'X', 'y', 'Y', 'z', 'Z' };
            int seed_count = seed.Length;
            Random rand = new Random();
            StringBuilder sb = new StringBuilder(4);
            for (int i = 0; i < d; i++)
                sb.Append(seed[rand.Next(0, seed_count)]);
            return sb.ToString();
        }

        /**//// <summary>
        ///  获取给定Url PageHtml
        /// </summary>
        /// <param name="url">Url</param>
        /// <returns>PageHtml</returns>
        public static string GetPageHTML(string url)
        ...{
            WebRequest request = null;
            HttpWebResponse response = null;
            Stream stream = null;
            StreamReader sr = null;

            try
            ...{
                request = WebRequest.Create(url);
                request.Credentials = CredentialCache.DefaultCredentials;
                request.Timeout = 2000;
                response = request.GetResponse() as HttpWebResponse;

                stream = response.GetResponseStream();
                sr = new StreamReader(stream, Encoding.Default);
                return sr.ReadToEnd();
            }
            catch
            ...{
                return string.Empty;
            }
            finally
            ...{
                if (sr != null)
                ...{
                    sr.Close();
                    sr.Dispose();
                }

                if (stream != null)
                ...{
                    stream.Close();
                    stream.Dispose();
                }
            }
        }

       
        

       

    }

}

<%...@ Page Language="C#" AutoEventWireup="true" CodeBehind="caiji01.aspx.cs" Inherits="MyTest.CaiJi.caiji01" validateRequest="false" %>

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

<html xmlns="http://www.w3.org/1999/xhtml" >
<head runat="server">
    <title>无标题页</title>
    <script type="text/javascript">...
        var xmlHttp;
        var key;
        function createXMLHttpRequest() ...{
            if (window.ActiveXObject) ...{
                xmlHttp = new ActiveXObject("Microsoft.XMLHTTP");
            } 
            else if (window.XMLHttpRequest) ...{
                xmlHttp = new XMLHttpRequest();                
            }
        }
        
        function pollServer() ...{ 
            createXMLHttpRequest();
            data = "load="+"";
            var url = "Loading.aspx";
            xmlHttp.open("POST", url, true);
            xmlHttp.setRequestHeader("Content-Type","application/x-www-form-urlencoded");
            xmlHttp.onreadystatechange = pollCallback;
            xmlHttp.send(data);
        }

        function pollCallback() ...{
            if (xmlHttp.readyState == 4) ...{
                if (xmlHttp.status == 200) ...{
                    var percent_complete = xmlHttp.responseText;
                    var progress = document.getElementById("progress");
                    var progressPersent = document.getElementById("progressPersent");
                     progress.style.width = percent_complete + "%";
                     progressPersent.innerHTML = percent_complete + "%";
                    if (percent_complete < 100) ...{
                        setTimeout("pollServer()", 2000);
                    } else ...{
                        document.getElementById("complete").innerHTML = "已生成完成!";

                    }
                }
            }
        } 
        
           
 function clearBar() ...{
   var progress_bar = document.getElementById("progressBar");
   var progressPersent = document.getElementById("progressPersent");
   var complete = document.getElementById("complete");
   //progress_bar.style.visibility = "visible"
   progressPersent.innerHTML = " ";
   complete.innerHTML = "开始生成！";
 }
 function next()
 ...{    
    pollServer();
    __doPostBack('Go','');
    return  false;
 }
    </script>


</head>
<body>
    <form id="form1" runat="server">
    <div>结果：<asp:Label
            ID="Label1" runat="server" Text=""></asp:Label><br />
            
        获取列表页面：<asp:TextBox ID="TextBox1" runat="server"></asp:TextBox>
        <asp:Button ID="Button1" runat="server" Text="下一步" OnClick="Button1_Click" />
        <asp:Panel runat="server" ID="Panel1" Visible="false">
        开始循环标记：<asp:TextBox Text="" TextMode="MultiLine" runat="server" ID="TextBox2"></asp:TextBox><br />
        结束循环结束：<asp:TextBox Text="" TextMode="MultiLine" runat="server" ID="TextBox3"></asp:TextBox><br />
        <asp:Button ID="Button3" runat="server" Text="下一步" OnClick="Button3_Click"  />
        </asp:Panel>
        <asp:Panel runat="server" ID="StringDo" Visible="false">
        开始标记：<asp:TextBox Text="" TextMode="MultiLine" runat="server" ID="StartUrlString"></asp:TextBox><br />
        结束标记：<asp:TextBox Text="" TextMode="MultiLine" runat="server" ID="EndUrlString"></asp:TextBox><br />
        <a href="javascript:next();">下一步</a><asp:Button ID="Button2" runat="server" Text="下一步" OnClick="Button2_Click" />
        <asp:LinkButton ID="Go" runat="server" Text="生成" OnClick="Go_Click"></asp:LinkButton>
        </asp:Panel>
    </div>
    <div id="progressBar" style="padding:0px;border:solid black 0px;visibility:hidden">
<table width="300" border="0" cellspacing="0" cellpadding="0"  align="center" >
  <tr>
    <td align="center" id="progressPersent" >0%</td>
  </tr>
  <tr >
    <td>
 <table width="100%" border="1" cellspacing="0" cellpadding="0" bordercolor="#000000">
  <tr>
    <td>
 <table width="1%" border="0" cellspacing="0" cellpadding="0" bgcolor="#FF0000" id="progress" height="20">
              <tr>
                <td> </td>
              </tr>
            </table></td>
  </tr>
</table>
</td>
  </tr>
  <tr>
    <td align="center" id="complete"></td>
  </tr>
</table>
</div>

    </form>
    <script type="text/javascript">...
    clearBar();
    </script>
</body>
</html>