获取网站中response.Headers["location"]，找到抓取页面

最新推荐文章于 2025-06-21 11:28:57 发布

原创最新推荐文章于 2025-06-21 11:28:57 发布 · 2.2w 阅读

1 ·

CC 4.0 BY-SA版权

通过两次HTTP请求，实现从指定网址获取包含特定关键词的网页内容，并处理跳转链接。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >


        private static string getPageContent(string tastKeyword)
        {
            string htmlcontent = "";
            HttpWebRequest request = null;
            HttpWebResponse response = null;
            string gethost = string.Empty;
            CookieContainer cc = new CookieContainer();
            string Cookiesstr = string.Empty;

            try
            {

                //第一次POST请求
                string post = @"formhash=59f68caf&srchtxt={0}&searchsubmit=yes";//模拟请求数据

                string postdata = string.Format(post, tastKeyword);
                string LoginUrl = "http://bbs.hc360.com/search.php?mod=forum";
                request = (HttpWebRequest)WebRequest.Create(LoginUrl);//实例化web访问类
                request.Method = "POST";//数据提交方式为POST
                //模拟头
                request.ContentType = "application/x-www-form-urlencoded";
                byte[] postdatabytes = Encoding.GetEncoding("gbk").GetBytes(postdata);
                request.ContentLength = postdatabytes.Length;
                request.Referer = "http://bbs.hc360.com/search.php?mod=forum";
                request.AllowAutoRedirect = false;
                request.CookieContainer = cc;
                request.KeepAlive = true;
                
                //提交请求
                Stream stream;
                stream = request.GetRequestStream();
                stream.Write(postdatabytes, 0, postdatabytes.Length);
                stream.Close();
                //接收响应
                response = (HttpWebResponse)request.GetResponse();
                //保存返回cookie
                response.Cookies = request.CookieContainer.GetCookies(request.RequestUri);
                CookieCollection cook = response.Cookies;
                string strcrook = request.CookieContainer.GetCookieHeader(request.RequestUri);
                Cookiesstr = strcrook;
                //取第一次GET跳转地址
                //StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("gb2312"));
                //string content = sr.ReadToEnd();
                response.Close();
            }
            catch (Exception)
            {
                //第一次POST出错；
            }
            try
            {
                string loction = response.Headers["location"];
                gethost = "http://bbs.hc360.com/" + loction; //第一次GET地址
                request = (HttpWebRequest)WebRequest.Create(gethost);
                request.Method = "GET";
                request.KeepAlive = true;
                request.Headers.Add("Cookie:" + Cookiesstr);
                request.CookieContainer = cc;
                request.AllowAutoRedirect = true;
                response = (HttpWebResponse)request.GetResponse();
                //设置cookie   
                Cookiesstr = request.CookieContainer.GetCookieHeader(request.RequestUri);
                //取再次跳转链接   
                StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("gbk"));
                htmlcontent = sr.ReadToEnd();
                request.Abort();
                sr.Close();
                response.Close();
            }
            catch (Exception)
            {
                //第二次POST出错   
            }
            return htmlcontent;
        }