asp.net 网页抓取内容

最新推荐文章于 2019-03-22 23:06:33 发布

qq_27915701

最新推荐文章于 2019-03-22 23:06:33 发布

阅读量355

点赞数

分类专栏： Asp.net 文章标签： asp.net 网页抓取内容

Asp.net 专栏收录该内容

13 篇文章

订阅专栏

本文介绍了一种从特定网站抓取船舶信息的方法，并通过正则表达式解析网页内容，提取所需数据。此外，还实现了去除HTML标签的功能，确保提取到的数据格式正确。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
//
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Text;

namespace WSYL.Web.Common
{
public static class GetSteamShipInfo
{
public static string GetWebSite(string steamshipname,int itype)
{
if (steamshipname == null || steamshipname.Trim() == "")
return null;
//step1: get html from url
string urlToCrawl = @"网址";
//generate http request
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(urlToCrawl);
//use GET method to get url's html
req.Method = "GET";
//use request to get response
HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
// 二〇一五年八月十二日 18:14:45 需要增加判断网页解析超时问题防止网页假死
// string htmlCharset = "UTF-8";
string htmlCharset = "utf-8";
//use songtaste's html's charset GB2312 to decode html
//otherwise will return messy code
Encoding htmlEncoding = Encoding.GetEncoding(htmlCharset);
StreamReader sr = new StreamReader(resp.GetResponseStream(), htmlEncoding);
//read out the returned html
string respHtml = sr.ReadToEnd();
//第三种获取内容
//Match TitleMatch = Regex.Match(rtbExtractedHtml.Text.ToString(), "<td width=\"30%\">([^<]*)</td>", RegexOptions.IgnoreCase | RegexOptions.Multiline);
//需要获取的代码开始和结尾内容

Match TitleMatch2 = Regex.Match(respHtml.ToString(), "<td align=\"left\" bgcolor=\"#EEEEEE\">([^<]*)</td>", RegexOptions.IgnoreCase | RegexOptions.Multiline);
// txbExtractedInfo.Text = TitleMatch2.Groups[1].Value+"/"+ TitleMatch2.Groups[2].Value;
if (TitleMatch2.Groups[1].Value.Length == 0 || TitleMatch2.Groups[1].Value=="")
return respHtml = "";
if(itype==0)
{
respHtml = TitleMatch2.Groups[1].Value.ToString();
}
if(itype==1)
{
respHtml = StripHtml(TitleMatch2.NextMatch().Value.ToString());
}
if (itype == 2)
{
respHtml = TitleMatch2.Groups[1].Value + "/" + StripHtml(TitleMatch2.NextMatch().Value.ToString());
}
return respHtml;
}
/// <summary>
/// 去除html标签和空格有些例外会使得去除不干净，所以建议连续两次转化。这样将Html标签转化为了空格。太多连续的空格会影响之后对字符串的操作
/// </summary>
/// <param name="strHtml">标签内容</param>
/// <returns></returns>
private static string StripHtml(string strHtml)
{
Regex objRegExp = new Regex("<(.|\n)+?>");
string strOutput = objRegExp.Replace(strHtml, "");
strOutput = strOutput.Replace("<", "<");
strOutput = strOutput.Replace(">", ">");
//把所有空格变为一个空格
Regex r = new Regex(@"\s+");
strOutput = r.Replace(strOutput, " ");
return strOutput.Trim();
}
}
}