C#实现抓取和分析网页类实例

/** Author:Sunjoy at CCNU* 如果您改进了这个类请发一份代码给我(ccnusjy 在gmail.com)*/using System;using System.Data;using System.Configuration;using System.Net;using System.IO;using System.Text;using System.Collections.Generic;using System.Text.RegularExpressions;using System.Threading;using System.Web;/// <summary>/// 网页类/// </summary>public class WebPage{ #region 私有成员 private Uri m_uri; //网址 private List<Link> m_links; //此网页上的链接 private string m_title;  //此网页的标题 private string m_html;   //此网页的HTML代码 private string m_outstr;  //此网页可输出的纯文本 private bool m_good;   //此网页是否可用 private int m_pagesize;  //此网页的大小 private static Dictionary<string, CookieContainer> webcookies = new Dictionary<string, CookieContainer>();//存放所有网页的Cookie private string m_post; //此网页的登陆页需要的POST数据 private string m_loginurl; //此网页的登陆页 #endregion #region 私有方法 /// <summary> /// 这私有方法从网页的HTML代码中分析出链接信息 /// </summary> /// <returns>List<Link></returns> private List<Link> getLinks() {  if (m_links.Count == 0)  {   Regex[] regex = new Regex[2];   regex[0] = new Regex("(?m)<a[^><]+href=(/"|')?(?<url>([^>/"'//s)])+)(/"|')?[^>]*>(?<text>(//w|//W)*?)</", RegexOptions.Multiline | RegexOptions.IgnoreCase);   regex[1] = new Regex("<[i]*frame[^><]+src=(/"|')?(?<url>([^>/"'//s)])+)(/"|')?[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase);   for (int i = 0; i < 2; i++)   {    Match match = regex[i].Match(m_html);    while (match.Success)    {     try     {      string url = new Uri(m_uri, match.Groups["url"].Value).AbsoluteUri;      string text = "";      if (i == 0) text = new Regex("(<[^>]+>)|(//s)|( )|&|/"", RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(match.Groups["text"].Value, "");      Link link = new Link(url, text);      m_links.Add(link);     }     catch(Exception ex){Console.WriteLine(ex.Message); };     match = match.NextMatch();    }   }  }  return m_links; } /// <summary> /// 此私有方法从一段HTML文本中提取出一定字数的纯文本 /// </summary> /// <param name="instr">HTML代码</param> /// <param name="firstN">提取从头数多少个字</param> /// <param name="withLink">是否要链接里面的字</param> /// <returns>纯文本</returns> private string getFirstNchar(string instr, int firstN, bool withLink) {  if (m_outstr == "")  {   m_outstr = instr.Clone() as string;   m_outstr = new Regex(@"(?m)<script[^>]*>(/w|/W)*?</script[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase ).Replace(m_outstr, "");   m_outstr = new Regex(@"(?m)<style[^>]*>(/w|/W)*?</style[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase ).Replace(m_outstr, "");   m_outstr = new Regex(@"(?m)<select[^>]*>(/w|/W)*?</select[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase ).Replace(m_outstr, "");   if (!withLink) m_outstr = new Regex(@"(?m)<a[^>]*>(/w|/W)*?</a[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(m_outstr, "");   Regex objReg = new System.Text.RegularExpressions.Regex("(<[^>]+?>)| ", RegexOptions.Multiline | RegexOptions.IgnoreCase);   m_outstr = objReg.Replace(m_outstr, "");   Regex objReg2 = new System.Text.RegularExpressions.Regex("(//s)+", RegexOptions.Multiline | RegexOptions.IgnoreCase);   m_outstr = objReg2.Replace(m_outstr, " ");  }  return m_outstr.Length > firstN ? m_outstr.Substring(0, firstN) : m_outstr; } /// <summary> /// 此私有方法返回一个IP地址对应的无符号整数 /// </summary> /// <param name="x">IP地址</param> /// <returns></returns> private uint getuintFromIP(IPAddress x) {  Byte[] bt = x.GetAddressBytes();  uint i = (uint)(bt[0] * 256 * 256 * 256);  i += (uint)(bt[1] * 256 * 256);  i += (uint)(bt[2] * 256);  i += (uint)(bt[3]);  return i; } #endregion #region 公有文法 /// <summary> /// 此公有方法提取网页中一定字数的纯文本，包括链接文字 /// </summary> /// <param name="firstN">字数</param> /// <returns></returns> public string getContext(int firstN) {  return getFirstNchar(m_html, firstN, true); } /// <summary> /// 此公有方法提取网页中一定字数的纯文本，不包括链接文字 /// </summary> /// <param name="firstN"></param> /// <returns></returns> public string getContextWithOutLink(int firstN) {  return getFirstNchar(m_html, firstN, false); } /// <summary> /// 此公有方法从本网页的链接中提取一定数量的链接，该链接的URL满足某正则式 /// </summary> /// <param name="pattern">正则式</param> /// <param name="count">返回的链接的个数</param> /// <returns>List<Link></returns> public List<Link> getSpecialLinksByUrl(string pattern,int count) {  if(m_links.Count==0)getLinks();  List<Link> SpecialLinks = new List<Link>();  List<Link>.Enumerator i;  i = m_links.GetEnumerator();  int cnt = 0;  while (i.MoveNext() && cnt<count)  {   if (new Regex(pattern, RegexOptions.Multiline | RegexOptions.IgnoreCase ).Match(i.Current.url).Success)   {    SpecialLinks.Add(i.Current);    cnt++;   }  }   return SpecialLinks; } /// <summary> /// 此公有方法从本网页的链接中提取一定数量的链接，该链接的文字满足某正则式 /// </summary> /// <param name="pattern">正则式</param> /// <param name="count">返回的链接的个数</param> /// <returns>List<Link></returns> public List<Link> getSpecialLinksByText(string pattern,int count) {  if (m_links.Count == 0) getLinks();  List<Link> SpecialLinks = new List<Link>();  List<Link>.Enumerator i;  i = m_links.GetEnumerator();  int cnt = 0;  while (i.MoveNext() && cnt < count)  {   if (new Regex(pattern, RegexOptions.Multiline | RegexOptions.IgnoreCase ).Match(i.Current.text).Success)   {    SpecialLinks.Add(i.Current);    cnt++;   }  }  return SpecialLinks; } /// <summary> /// 此公有方法获得所有链接中在一定IP范围的链接 /// </summary> /// <param name="_ip_start">起始IP</param> /// <param name="_ip_end">终止IP</param> /// <returns></returns> public List<Link> getSpecialLinksByIP(string _ip_start, string _ip_end) {  IPAddress ip_start = IPAddress.Parse(_ip_start);  IPAddress ip_end = IPAddress.Parse(_ip_end);  if (m_links.Count == 0) getLinks();  List<Link> SpecialLinks = new List<Link>();  List<Link>.Enumerator i;  i = m_links.GetEnumerator();  while (i.MoveNext())  {   IPAddress ip;   try   {    ip = Dns.GetHostEntry(new Uri(i.Current.url).Host).AddressList[0];   }   catch { continue; }   if(getuintFromIP(ip)>=getuintFromIP(ip_start) && getuintFromIP(ip)<=getuintFromIP(ip_end))   {    SpecialLinks.Add(i.Current);   }  }  return SpecialLinks; } /// <summary> /// 这公有方法提取本网页的纯文本中满足某正则式的文字 /// </summary> /// <param name="pattern">正则式</param> /// <returns>返回文字</returns> public string getSpecialWords(string pattern) {  if (m_outstr == "") getContext(Int16.MaxValue);  Regex regex = new Regex(pattern, RegexOptions.Multiline | RegexOptions.IgnoreCase );  Match mc=regex.Match(m_outstr);  if (mc.Success)   return mc.Groups[1].Value;  return string.Empty; } #endregion #region 构造函数 private void Init(string _url) {  try  {   m_uri = new Uri(_url);   m_links = new List<Link>();   m_html = "";   m_outstr = "";   m_title = "";   m_good = true;   if (_url.EndsWith(".rar") || _url.EndsWith(".dat") || _url.EndsWith(".msi"))   {    m_good = false;    return;   }   HttpWebRequest rqst = (HttpWebRequest)WebRequest.Create(m_uri);   rqst.AllowAutoRedirect = true;   rqst.MaximumAutomaticRedirections = 3;   rqst.UserAgent = "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)";   rqst.KeepAlive = true;   rqst.Timeout = 30000;   lock (WebPage.webcookies)   {    if (WebPage.webcookies.ContainsKey(m_uri.Host))     rqst.CookieContainer = WebPage.webcookies[m_uri.Host];    else    {     CookieContainer cc = new CookieContainer();     WebPage.webcookies[m_uri.Host] = cc;     rqst.CookieContainer = cc;    }   }   HttpWebResponse rsps = (HttpWebResponse)rqst.GetResponse();   Stream sm = rsps.GetResponseStream();   if (!rsps.ContentType.ToLower().StartsWith("text/") || rsps.ContentLength > 1 << 22)   {    rsps.Close();    m_good = false;    return;   }   Encoding cding = System.Text.Encoding.Default;   string contenttype=rsps.ContentType.ToLower();   int ix = contenttype.IndexOf("charset=");   if (ix != -1)   {    try    {     cding = System.Text.Encoding.GetEncoding(rsps.ContentType.Substring(ix + "charset".Length + 1));    }    catch    {     cding = Encoding.Default;    }    m_html = new StreamReader(sm, cding).ReadToEnd();   }   else   {    m_html = new StreamReader(sm, cding).ReadToEnd();    Regex regex = new Regex("charset=(?<cding>[^=]+)?/"",RegexOptions.IgnoreCase);    string strcding = regex.Match(m_html).Groups["cding"].Value;    try    {     cding = Encoding.GetEncoding(strcding);    }    catch{     cding = Encoding.Default;    }    byte[] bytes=Encoding.Default.GetBytes(m_html.ToCharArray());    m_html = cding.GetString(bytes);    if (m_html.Split('?').Length > 100)    {     m_html=Encoding.Default.GetString(bytes);    }   }      m_pagesize = m_html.Length;   m_uri = rsps.ResponseUri;   rsps.Close();  }  catch (Exception ex)  {   Console.WriteLine(ex.Message+m_uri.ToString());   m_good = false;  } } public WebPage(string _url) {  string uurl = "";  try  {   uurl = Uri.UnescapeDataString(_url);   _url = uurl;  }  catch { };  Regex re = new Regex("(?<h>[^/x00-/xff]+)");  Match mc = re.Match(_url);  if (mc.Success)  {   string han = mc.Groups["h"].Value;   _url = _url.Replace(han, System.Web.HttpUtility.UrlEncode(han, Encoding.GetEncoding("GB2312")));  }  Init(_url); } public WebPage(string _url, string _loginurl, string _post) {  string uurl = "";  try  {   uurl = Uri.UnescapeDataString(_url);   _url = uurl;  }  catch { };  Regex re = new Regex("(?<h>[^/x00-/xff]+)");  Match mc = re.Match(_url);  if (mc.Success)  {   string han = mc.Groups["h"].Value;   _url = _url.Replace(han, System.Web.HttpUtility.UrlEncode(han, Encoding.GetEncoding("GB2312")));  }  if (_loginurl.Trim() == "" || _post.Trim() == "" || WebPage.webcookies.ContainsKey(new Uri(_url).Host))  {   Init(_url);  }  else  {   #region 登陆   string indata = _post;   m_post = _post;   m_loginurl = _loginurl;   byte[] bytes = Encoding.Default.GetBytes(_post);   CookieContainer myCookieContainer = new CookieContainer();   try   {    //新建一个CookieContainer来存放Cookie集合     HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(_loginurl);    //新建一个HttpWebRequest     myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";    myHttpWebRequest.AllowAutoRedirect = false;    myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)";    myHttpWebRequest.Timeout = 60000;    myHttpWebRequest.KeepAlive = true;    myHttpWebRequest.ContentLength = bytes.Length;    myHttpWebRequest.Method = "POST";    myHttpWebRequest.CookieContainer = myCookieContainer;    //设置HttpWebRequest的CookieContainer为刚才建立的那个myCookieContainer     Stream myRequestStream = myHttpWebRequest.GetRequestStream();    myRequestStream.Write(bytes, 0, bytes.Length);    myRequestStream.Close();    HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();    foreach (Cookie ck in myHttpWebResponse.Cookies)    {     myCookieContainer.Add(ck);    }    myHttpWebResponse.Close();   }   catch   {    Init(_url);    return;   }   #endregion   #region 登陆后再访问页面   try   {    m_uri = new Uri(_url);    m_links = new List<Link>();    m_html = "";    m_outstr = "";    m_title = "";    m_good = true;    if (_url.EndsWith(".rar") || _url.EndsWith(".dat") || _url.EndsWith(".msi"))    {     m_good = false;     return;    }    HttpWebRequest rqst = (HttpWebRequest)WebRequest.Create(m_uri);    rqst.AllowAutoRedirect = true;    rqst.MaximumAutomaticRedirections = 3;    rqst.UserAgent = "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)";    rqst.KeepAlive = true;    rqst.Timeout = 30000;    rqst.CookieContainer = myCookieContainer;    lock (WebPage.webcookies)    {     WebPage.webcookies[m_uri.Host] = myCookieContainer;    }    HttpWebResponse rsps = (HttpWebResponse)rqst.GetResponse();    Stream sm = rsps.GetResponseStream();    if (!rsps.ContentType.ToLower().StartsWith("text/") || rsps.ContentLength > 1 << 22)    {     rsps.Close();     m_good = false;     return;    }    Encoding cding = System.Text.Encoding.Default;    int ix = rsps.ContentType.ToLower().IndexOf("charset=");    if (ix != -1)    {     try     {      cding = System.Text.Encoding.GetEncoding(rsps.ContentType.Substring(ix + "charset".Length + 1));     }     catch     {      cding = Encoding.Default;     }    }    m_html = new StreamReader(sm, cding).ReadToEnd();    m_pagesize = m_html.Length;    m_uri = rsps.ResponseUri;    rsps.Close();   }   catch (Exception ex)   {    Console.WriteLine(ex.Message+m_uri.ToString());    m_good = false;   }   #endregion  } } #endregion #region 属性 /// <summary> /// 通过此属性可获得本网页的网址，只读 /// </summary> public string URL {  get  {   return m_uri.AbsoluteUri;  } } /// <summary> /// 通过此属性可获得本网页的标题，只读 /// </summary> public string Title {  get  {   if (m_title == "")   {    Regex reg = new Regex(@"(?m)<title[^>]*>(?<title>(?:/w|/W)*?)</title[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase );    Match mc = reg.Match(m_html);    if (mc.Success)     m_title= mc.Groups["title"].Value.Trim();   }   return m_title;  } } /// <summary> /// 此属性获得本网页的所有链接信息，只读 /// </summary> public List<Link> Links {  get  {   if (m_links.Count == 0) getLinks();   return m_links;  } } /// <summary> /// 此属性返回本网页的全部纯文本信息，只读 /// </summary> public string Context {  get  {   if (m_outstr == "") getContext(Int16.MaxValue);   return m_outstr;  } } /// <summary> /// 此属性获得本网页的大小 /// </summary> public int PageSize {  get  {   return m_pagesize;  } } /// <summary> /// 此属性获得本网页的所有站内链接 /// </summary> public List<Link> InsiteLinks {  get  {   return getSpecialLinksByUrl("^http://"+m_uri.Host,Int16.MaxValue);  } } /// <summary> /// 此属性表示本网页是否可用 /// </summary> public bool IsGood {  get  {   return m_good;  } } /// <summary> /// 此属性表示网页的所在的网站 /// </summary> public string Host {  get  {   return m_uri.Host;  } } /// <summary> /// 此网页的登陆页所需的POST数据 /// </summary> public string PostStr {  get  {   return m_post;  } } /// <summary> /// 此网页的登陆页 /// </summary> public string LoginURL {  get  {   return m_loginurl;  } } #endregion}/// <summary>/// 链接类/// </summary>public class Link{ public string url; //链接网址 public string text; //链接文字 public Link(string _url, string _text) {  url = _url;  text = _text; }}