以前常听到别人做采集程序,今天俺也小试身手。。
using system;
using system.collections;
using system.componentmodel;
using system.data;
using system.drawing;
using system.web;
using system.web.sessionstate;
using system.web.ui;
using system.net;
using system.text;
using system.text.regularexpressions;
using system.io;
using system.data.sqlclient;
using system.web.ui.webcontrols;
using system.web.ui.htmlcontrols;
namespace temp
{
/// <summary>
/// gethtmlsourcefromurl 的摘要说明。
/// </summary>
public class gethtmlsourcefromurl : system.web.ui.page
{
private sqlconnection con;
private void page_load(object sender, system.eventargs e)
{
// 在此处放置用户代码以初始化页面
if(!ispostback)
{
inserttodb();
}
}
private void inserttodb()
{
for(int i=1;i<1000;i++)
{
string html = string.empty;
string sqltext="insert into beijingbus(buslinenumber,html) values(@bln,@html)";
string [email protected]"server=.xxxx;user id=xxxx;pwd=xxxxx;database=map";
con =new sqlconnection(connectionstring);
sqlcommand cmd = new sqlcommand(sqltext,con);
string [email protected]"http://beijing.ibusdb.com/?busline="+i+"&s=busline&x=31&y=18"; // ^_^,这是取北京的 转到首页,换换参数,就可以取其它城市的了
string content = string.empty;
httpwebrequest request = (httpwebrequest)webrequest.create(url);
httpwebresponse response =(httpwebresponse) request.getresponse();
streamreader stream = new streamreader(response.getresponsestream(),encoding.getencoding("gb2312"));
content =stream.readtoend();
stream.close();
response.close();
int start=3487; //去广告 取正文
int end= content.length-4222;//去广告 取正文
if(end-start>0) //判断存在第i路公交
{
html=regex.replace(content.substring(start,end-start),@"<[^>]+>",""); //分离html代码
try
{
cmd.parameters.add("@bln",sqldbtype.int).value =i;
cmd.parameters.add("@html",sqldbtype.text).value= html;
con.open();
cmd.executenonquery();
}
catch(sqlexception err)
{
response.write(err.message);
con.close();
break;
}
con.close();
}
}
}
--表sql脚本:
create table [dbo].[beijingbus](
[id] [int] identity(1,1) not null,
[buslinenumber] [int] null,
[html] [nvarchar](max) collate chinese_prc_ci_as null
) on [primary]
新闻热点
疑难解答
图片精选