新闻采集

//此方法获取远程地址的源代码
        public static string GetHtml(string url)
        {
                try
                {
                        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(url);
                        HttpWebResponse webResponse = (HttpWebResponse)webRequest.GetResponse();
                        Stream stream = webResponse.GetResponseStream();
                        System.IO.StreamReader streamReader = new StreamReader(stream, System.Text.Encoding.GetEncoding("GB2312"));
                        string content = streamReader.ReadToEnd();
                        streamReader.Close();
                        webResponse.Close();
                        return content.Trim();
                }
                catch
                {
                        return "-1";
                }

        }
//按钮
protected void btnRead_Click(object sender, EventArgs e)
        {
                string rssURL = txtUrl.Text.Trim();//列表页面url 地址
                string resultlist = GetHtml(rssURL);//列表页面数据
Return(resultlist, int.Parse(TextBox7.Text.ToString()));//调用采集主要方法
        }
//主体方法
private string Return(string result, int j)
        {
                string strReg = TextBox1.Text.Trim() + @"[(\s\S)]*?" + TextBox2.Text.Trim();//输入新闻列表开始结尾-- /news/2008 ---.shtml
                Regex re = new Regex(strReg, RegexOptions.IgnoreCase);
                Match m = re.Match(result);
                int mn = 0;
                if (i <= j)
                {
                        string NewsString = GetHtml(this.TextBox8.Text.Trim() + m.Value);//查询新闻内容数据

                        string rg_title = TextBox5.Text.Trim() + @"([\s\S]*?)" + TextBox6.Text.Trim();//新闻标题标签
                        Regex re_title = new Regex(rg_title, RegexOptions.IgnoreCase);
                        Match m_title = re_title.Match(NewsString);
                        string n_title = System.Text.RegularExpressions.Regex.Replace(m_title.Value.ToString().Replace("'", ""), "<[^>]+>", "");//新闻标题

                        string rg_note = TextBox3.Text.Trim() + @"([\s\S]*?)" + TextBox4.Text.Trim();//新闻内容标签
                        Regex re_note = new Regex(rg_note, RegexOptions.IgnoreCase);
                        Match m_note = re_note.Match(NewsString);
                        string n_note = m_note.Value.ToString().Replace("'", "");//新闻内容
                        string selecttitle = "select n_title from zy_News where n_title='" + FunctionLib.StrLib.cutStr(n_title, 48) + "'";
                        DataSet dstitle = SqlHelp.ExecuteDataSet(SqlHelp.sqlCon, CommandType.Text, selecttitle, null);
                        if (dstitle.Tables[0].Rows.Count == 0)
                        {
                                if (n_note != "" && n_note != null)
                                {
                                        string insertsql = "insert into zy_News values(58,0,'" + n_title + "','','','" + n_note + "','','',1,'" + n_title + "','',1,'','',0,0,getdate(),getdate(),0,0,0,0,getdate())";

                                        SqlHelp.ExecuteNonQuery(SqlHelp.sqlCon, CommandType.Text, insertsql, null);
                                        mn++;
                                        i++;
                                }
                        }
                        result1 = result.Replace(m.Value, " ");
                        Return(result1, j);//递归
                }

        }

转载于:https://blog.51cto.com/soholi/158759