本文介绍了多种网页抓取方法，包括使用代理的HTML获取、简单页面抓取及通过WebBrowser控件抓取已完成加载的页面内容。同时，还提供了文件下载及保存的实现方式。
是程序中时不时用到的 还没有整理出来 等回头在整理一份
用代理账号密码获取页面的
/// <summary>
/// 代理返回Html Get
/// </summary>
/// <param name="strReferUrl">来源url</param>
/// <param name="bEnableProxy">是否用代理</param>
/// <param name="strProxyHost">代理地址</param>
/// <param name="iProxyPort">代理端口号</param>
/// <param name="strName">代理账号</param>
/// <param name="strPwd">代理密码</param>
/// <param name="url">url</param>
/// <param name="SaveCookies">是否保存cookies</param>
///<param name="OutTime">暂停时间</param>
/// <returns></returns>
public string GetProxyHtml(string strReferUrl, bool bEnableProxy, string strProxyHost, int iProxyPort, string strName, string strPwd, string url, bool SaveCookies, int? time = 0, int? isCook = 0)
{
try
{
HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
if (bEnableProxy)
{
myHttpWebRequest.Proxy = new WebProxy(strProxyHost, iProxyPort)
{
Credentials = new NetworkCredential(strName, strPwd)
};
myHttpWebRequest.Headers.Add("Proxy-Authorization", "Basic MzUxMTM6MDM1NjI5");
myHttpWebRequest.UseDefaultCredentials = true;
}
myHttpWebRequest.ServicePoint.Expect100Continue = true;
myHttpWebRequest.MaximumAutomaticRedirections = 50;
myHttpWebRequest.Method = "GET";
myHttpWebRequest.Headers.Add("Accept-Language", "zh-Hans-CN,zh-Hans;q=0.5");
// myHttpWebRequest.Host = "jcr.incites.thomsonreuters.com";
myHttpWebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0";
myHttpWebRequest.KeepAlive = true;
myHttpWebRequest.ContentType = "text/html; charset=utf-8";
myHttpWebRequest.AllowAutoRedirect = true;
if (!string.IsNullOrEmpty(strReferUrl))
{
myHttpWebRequest.Referer = strReferUrl;
}
if (isCook != 0)
{
myHttpWebRequest.CookieContainer = new CookieContainer();
}
else
{
if (cookies == null)
{
myHttpWebRequest.CookieContainer = new CookieContainer();
}
else
{
myHttpWebRequest.CookieContainer = cookies;
}
}
myHttpWebRequest.Timeout = 32000;
myHttpWebRequest.ReadWriteTimeout = 32000;
HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();
myHttpWebResponse.Cookies = cookies.GetCookies(myHttpWebRequest.RequestUri);
Stream myStream = myHttpWebResponse.GetResponseStream();
StreamReader myStreamReader = new StreamReader(myStream, Encoding.GetEncoding("utf-8"));
if (SaveCookies)
{
cookies = myHttpWebRequest.CookieContainer;
}
var html = myStreamReader.ReadToEnd();
if (time != 0)
{
Thread.Sleep(5000);
}
myStream.Close();
myStreamReader.Close();
myHttpWebResponse.Close();
myHttpWebRequest.Abort();
return html.Replace("\n", "").Replace("\r", "");
}
catch (Exception e)
{
return "";
}
}
简单的页面
static string GetWebClient(string url)
{
string strHTML = "";
WebClient myWebClient = new WebClient();
Stream myStream = myWebClient.OpenRead(url);
StreamReader sr = new StreamReader(myStream, System.Text.Encoding.GetEncoding("utf-8"));
strHTML = sr.ReadToEnd();
myStream.Close();
return strHTML;
}
static string GetWebRequest(string url)
{
Uri uri = new Uri(url);
WebRequest myReq = WebRequest.Create(uri);
WebResponse result = myReq.GetResponse();
Stream receviceStream = result.GetResponseStream();
StreamReader readerOfStream = new StreamReader(receviceStream, System.Text.Encoding.GetEncoding("utf-8"));
string strHTML = readerOfStream.ReadToEnd();
readerOfStream.Close();
receviceStream.Close();
result.Close();
return strHTML;
}
static string GetHttpWebRequest(string url)
{
Uri uri = new Uri(url);
HttpWebRequest myReq = (HttpWebRequest)WebRequest.Create(uri);
myReq.Host = "tandfonline.com";
myReq.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0";
myReq.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
myReq.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
myReq.KeepAlive = true;
HttpWebResponse result = (HttpWebResponse)myReq.GetResponse();
Stream receviceStream = result.GetResponseStream();
StreamReader readerOfStream = new StreamReader(receviceStream, System.Text.Encoding.GetEncoding("utf-8"));
string strHTML = readerOfStream.ReadToEnd();
readerOfStream.Close();
receviceStream.Close();
result.Close();
return strHTML;
}
static string GetContentFromUrl(string URL)
{
try
{
string strBuff = "";
int byteRead = 0;
char[] cbuffer = new char[256];
HttpWebRequest httpReq = (HttpWebRequest)WebRequest.Create(new Uri(URL));
HttpWebResponse httpResp = (HttpWebResponse)httpReq.GetResponse();
Stream respStream = httpResp.GetResponseStream();
StreamReader respStreamReader = new StreamReader(respStream, System.Text.Encoding.UTF8);
byteRead = respStreamReader.Read(cbuffer, 0, 256);
while (byteRead != 0)
{
string strResp = new string(cbuffer, 0, byteRead);
strBuff = strBuff + strResp;
byteRead = respStreamReader.Read(cbuffer, 0, 256);
}
respStream.Close();
return strBuff;
}
catch (Exception ex)
{
return ex.Message;
}
}
static string GetContentFromUrl1(string url)
{
try
{
WebClient client = new WebClient();
client.Credentials = CredentialCache.DefaultCredentials;//获取或设置请求凭据
Byte[] pageData = client.DownloadData(url); //下载数据
string pageHtml = System.Text.Encoding.UTF8.GetString(pageData);
return pageHtml;
}
catch (WebException ex)
{
return ex.Message;
}
}
static string GetStringByUrl(string Url)
{
if (Url.Equals("about:blank")) return null; ;
if (!Url.StartsWith("http://") && !Url.StartsWith("https://")) { Url = "http://" + Url; }
int dialCount = 0;
loop:
StreamReader sreader = null;
string result = string.Empty;
try
{
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(Url);
//httpWebRequest.Timeout = 20;
httpWebRequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)";
httpWebRequest.Accept = "*/*";
httpWebRequest.KeepAlive = true;
httpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
if (httpWebResponse.StatusCode == HttpStatusCode.OK)
{
sreader = new StreamReader(httpWebResponse.GetResponseStream(), System.Text.Encoding.GetEncoding("utf-8"));
char[] cCont = new char[256];
int count = sreader.Read(cCont, 0, 256);
while (count > 0)
{ // Dumps the 256 characters on a string and displays the string to the console.
String str = new String(cCont, 0, count);
result += str;
count = sreader.Read(cCont, 0, 256);
}
}
if (null != httpWebResponse) { httpWebResponse.Close(); }
return result;
}
catch (WebException e)
{
if (e.Status == WebExceptionStatus.ConnectFailure)
{
dialCount++;
ReDial();
}
if (dialCount < 5) { goto loop; }
return null;
}
finally { if (sreader != null) { sreader.Close(); } }
}
下面的方法是调用WebBrowser 是为了页面加载完成后在取数据！ 提示：据说此代码运行没有前面的快 所以不到需要的时候不用
[STAThread]
static void Main(string[] args)
[STAThread]
static void Main(string[] args)
{
var html = GetPageStringbyWebBrowser(url);
}
private static string GetPageStringbyWebBrowser(string url)
{
string htmlstr = "";
if (url.Equals("about:blank")) htmlstr = "";
if (!url.StartsWith("http://") && !url.StartsWith("https://")) { url = "http://" + url; }
WebBrowser myWB = new WebBrowser();
myWB.ScrollBarsEnabled = false;
myWB.Navigate(url.ToString());
while (myWB.ReadyState != WebBrowserReadyState.Complete)
{
System.Windows.Forms.Application.DoEvents();
}
if (myWB != null)
{
System.IO.StreamReader getReader = null;
try
{
getReader = new System.IO.StreamReader(myWB.DocumentStream, System.Text.Encoding.GetEncoding(myWB.Document.Encoding));
htmlstr = getReader.ReadToEnd();
}
catch { htmlstr = ""; }
finally
{
if (getReader != null) { getReader.Close(); }
myWB.Dispose();
}
}
return htmlstr;
}
保存文件 第一个是常用的：
如果报错 比如说
请求被中止: 未能创建 SSL/TLS 安全通道
用第二个方法
static bool GetAndSavePdf(string url, string path)
{
HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
myHttpWebRequest.Timeout = 20 * 1000; //连接超时
myHttpWebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0";
//myHttpWebRequest.CookieContainer = cookies; //使用已经保存的cookies
HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();
Stream stream = myHttpWebResponse.GetResponseStream();
if (stream != null)
{
SaveFile(path, stream);
return true;
}
return false;
}
public static void SaveFile(string path, Stream stream)
{
int BufSize = 102400;
if (System.IO.File.Exists(path))
return;
FileInfo file = new FileInfo(path);
if (!System.IO.Directory.Exists(file.DirectoryName))
{
System.IO.Directory.CreateDirectory(file.DirectoryName);
}
FileStream fs = new FileStream(path, FileMode.Create);
byte[] buf = new byte[BufSize];
int size = 0;
try
{
size = stream.Read(buf, 0, BufSize);
while (size > 0)
{
fs.Write(buf, 0, size);
size = stream.Read(buf, 0, BufSize);
}
stream.Close();
}
catch (Exception ex)
{
throw ex;
}
finally { fs.Close(); }
}
第二个 保存文件
[DllImport("urlmon.dll", CharSet = CharSet.Auto, SetLastError = true)]
static extern Int32 URLDownloadToFile(
[MarshalAs(UnmanagedType.IUnknown)] object callerPointer,
[MarshalAs(UnmanagedType.LPWStr)] string url,
[MarshalAs(UnmanagedType.LPWStr)] string filePathWithName,
Int32 reserved,
IntPtr callBack);
public static FileInfo DownloadFile(string url, string destinationFullPathWithName)
{
URLDownloadToFile(null, url, destinationFullPathWithName, 0, IntPtr.Zero);
return new FileInfo(destinationFullPathWithName);
}
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
