//allow auto redirects from redirect headers request.allowautoredirect=true;
//maximum of 10 auto redirects request.maximumautomaticredirections=10;
//30 second timeout for request request.timeout=(int) new timespan(0,0,60).totalmilliseconds;
//give the crawler a name. request.useragent = "mozilla/4.0 (compatible; msie 6.0; windows nt 5.1)"; //request.useragent="mozilla/3.0 (compatible; my browser/1.0)";