用C#写asp.net搜索蜘蛛代码程序

最新推荐文章于 2022-07-12 09:35:53 发布

weixin_30326515

最新推荐文章于 2022-07-12 09:35:53 发布

阅读量95

点赞数

CC 4.0 BY-SA版权

原文链接：http://www.cnblogs.com/pengxianwei/archive/2009/03/14/1411222.html

本文介绍了一种通过分析HTTP用户代理字符串来判断访问者是否为搜索引擎爬虫的方法，并记录爬虫信息。此外，还展示了如何针对爬虫展示特定链接。

protected bool robot()
    {
        bool brtn = false;
        string king_robots = "Baiduspider+@Baidu|Googlebot@Google|ia_archiver@Alexa|IAArchiver@Alexa|ASPSeek@ASPSeek|YahooSeeker@Yahoo|sohu-search@Sohu|help.yahoo.com/help/us/ysearch/slurp@Yahoo|sohu-search@SOHU|MSNBOT@MSN";
        string ls_spr;

        ls_spr=Request.ServerVariables["http_user_agent"].ToString();
        char[] delimiterChars = { '|' };
        char[] x = { '@' };
        string[] I1 = king_robots.Split(delimiterChars);

        for (int i = 0; i < I1.Length; i++)
        {
            string[] spider = I1[i].Split(x);
            if (ls_spr.IndexOf(spider[0].ToString()) > -1)
            {
                brtn = true;
                logrobots(spider[1].ToString());
                break;
            }
        }
        return brtn;
    }
    protected void logrobots(string spr)
    {

        spr = DateTime.Now.ToString() + "|" + spr;
        StreamWriter sw;
        try
        {
           sw = new StreamWriter(Server.MapPath("log.txt"), true, System.Text.Encoding.GetEncoding("gb2312")); //参数为false重写，为true在后面加入               sw.WriteLine(spr);
            sw.Close();
        }
        finally
        {

        }

    }
完成。

下面的为在别的网站上加链接。只有蜘蛛看到。
protected void addcontent()
    {
        StreamReader sr ;
        try
        {
            sr = new StreamReader(Server.MapPath("logurl.txt"), System.Text.Encoding.GetEncoding("gb2312"));
            String input;
            char[] UPer = { '|' };
            char[] xx = { '@' };
            while ((input = sr.ReadLine()) != null)
            {

                sb = sb + "<p>";
                string[] url = input.Split(UPer);
                for (int i = 0; i < url.Length; i++)
                {
                    string[] tt = url[i].Split(xx);
                    if (tt[0].Length > 0)
                    {
                        sb = sb + "<a href=" + tt[0].ToString() + " target=_blank>" + tt[1].ToString() + "</a>|";
                    }
                }
                sb = sb + "</p>";
            }
            sr.Close();

        }
        finally
        {

}

转载于:https://www.cnblogs.com/pengxianwei/archive/2009/03/14/1411222.html