获取网页中的所有超链接

最新推荐文章于 2021-08-27 12:26:42 发布

原创最新推荐文章于 2021-08-27 12:26:42 发布 · 2.5k 阅读

0 ·

CC 4.0 BY-SA版权

C#高级语法专栏收录该内容

11 篇文章

订阅专栏

本篇博客介绍了一个使用C#实现的简单应用程序，该程序能够从指定的网页中抓取并解析出所有的超链接。通过分析网页源代码并应用正则表达式来匹配超链接，最终将抓取到的有效链接输出到一个列表中。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Windows.Forms;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading;
using System.Net.NetworkInformation;

namespace frmExpendFlow
{
public partial class UIExpendFlow : Form
{
public UIExpendFlow()
{
InitializeComponent();
}
private static UIExpendFlow formInstance = null; //本窗体的实例(单例模式)

List<string> linkList = new List<string>();
string MyLink { get; set; } //链接对象
string MyScheme { get; set; } //超文本传输协议
bool MyAbsolute { get; set; } //是否符合指定协议链接

public static UIExpendFlow GetInstanceObject() //保证同一个窗体只实例化一次
{
if (formInstance == null)
{
formInstance = new UIExpendFlow();
formInstance.Show();
}
return formInstance;
}

private void UIExpendFlow_FormClosing(object sender, FormClosingEventArgs e) //实例置为空
{
formInstance = null;
}

private void UIExpendFlow_Activated(object sender, EventArgs e) //网址池输入框获取焦点
{
tbUrlPool.Focus();
}

private void btnClose_Click(object sender, EventArgs e) //关闭窗口
{
this.Close();
}

private void chkUrlPool_CheckStateChanged(object sender, EventArgs e) //按钮是否可用
{
if (chkUrlPool.Checked == true)
{
btnGetHyperlink.Enabled = true;
}
else
{
btnGetHyperlink.Enabled = false;
}
}

private void btnGetHyperlink_Click(object sender, EventArgs e) // 创建一个线程，用来执行委托
{
btnGetHyperlink.Enabled = false;
int UnableAdapters=0; //不可用网络适配器个数
int PCIAdapters = 0; //PCI适配器个数
NetworkInterface[] adapters = NetworkInterface.GetAllNetworkInterfaces(); //获取本地计算机上网络接口的对象
foreach (NetworkInterface adapter in adapters)
{
if (adapter.Description.IndexOf("PCI") > 0)
{
PCIAdapters++;
if (adapter.OperationalStatus.ToString().Equals("Up")) //Up网络接口已运行，可以传输数据包。
{
labLinkInfo.Text =string.Format("适配器{0}已连接至网络",adapter.Description);
labLinkInfo.Text = "正在获取超链接，请等待...";
Thread getHtmlThread = new Thread(CrossThreadFlush); //开始执行获取超链接的线程
getHtmlThread.IsBackground = true;
getHtmlThread.Start();
}
else
{
UnableAdapters++;
}
}
else
{
continue;
}
}
if (UnableAdapters == PCIAdapters)
{
MessageBox.Show("电脑未连接网络", "警告", MessageBoxButtons.OK, MessageBoxIcon.Warning);
btnGetHyperlink.Enabled = true;
return;
}
}
private void CrossThreadFlush() //声明一个委托，执行获取网页内容的方法
{
MethodInvoker In = new MethodInvoker(GetHtml);
this.BeginInvoke(In);
}
private void GetHtml() //获取网页的方法
{
if (txtAccessAddress.TextLength > 0)
{
//获取网页内容
linkList.Clear(); //清空list
tbUrlPool.Clear(); //清空网址池显示
Uri uri = new Uri(txtAccessAddress.Text.Trim()); //获取统一资源定位符
HttpWebRequest myReq = (HttpWebRequest)WebRequest.Create(uri); //实例化HttpWebRequest类
myReq.UserAgent = "User-Agent:Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36 LBBROWSER"; //浏览器标识（UA）
myReq.Accept = "*/*";
myReq.KeepAlive = true; //保持永久性连接
myReq.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
try
{
HttpWebResponse result = (HttpWebResponse)myReq.GetResponse();
Stream receviceStream = result.GetResponseStream();
StreamReader readerOfStream = new StreamReader(receviceStream, System.Text.Encoding.GetEncoding("utf-8"));
string strHTML = readerOfStream.ReadToEnd();
readerOfStream.Close();
receviceStream.Close();
result.Close();
String MyHrefPattern = @"<a[^>]+href=\s*(?:'(?<href>[^']+)'|""(?<href>[^""]+)""|(?<href>[^>\s]+))\s*[^>]*>(?<text>.*?)</a>"; //提取超链接的正则表达式
Regex MyHrefRegex = new Regex(MyHrefPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled);
Match MyHrefMatch = MyHrefRegex.Match(strHTML);
while (MyHrefMatch.Success)
{
MyLink = MyHrefMatch.Groups[1].Value.Replace("\r", "").Replace("\n", "");
if (MyLink.Substring(0, 1) != "#")//忽略书签，查找超级链接
{
MyAbsolute = false;
if (MyLink.Length > 8)
{
MyScheme = Uri.UriSchemeHttp; //http协议链接
if (MyLink.Substring(0, MyScheme.Length) == MyScheme)
MyAbsolute = true;
MyScheme = Uri.UriSchemeHttps;
if (MyLink.Substring(0, MyScheme.Length) == MyScheme)
MyAbsolute = true;
}
if (MyAbsolute)
{
if (linkList.Contains(string.Format("{0}\r\n", MyLink))) //判断链接是否重复
{
MyHrefMatch = MyHrefMatch.NextMatch();//继续查找超级链接
}
else
{
linkList.Add(string.Format("{0}\r\n", MyLink));
}
}
}
MyHrefMatch = MyHrefMatch.NextMatch();//继续查找超级链接
if (linkList.Count == 500) //设置获取的超链接的最大数
{
break;
}
}
myReq.KeepAlive = false; //断开连接
}
catch { }
foreach (string str in linkList)
{
tbUrlPool.AppendText(str);
}
labLinkInfo.Text = string.Format("{0} 网页中{1}个链接获取完毕\r", txtAccessAddress.Text, linkList.Count.ToString());
}
else
{
MessageBox.Show("访问地址不能不能为空，请输入访问地址", "提示", MessageBoxButtons.OK, MessageBoxIcon.Asterisk);
return;
}
btnGetHyperlink.Enabled = true;
}

}
}