Python制作爬虫采集小说

import urllib.requestimport http.cookiejarimport socketimport timeimport retimeout = 20socket.setdefaulttimeout(timeout)sleep_download_time = 10time.sleep(sleep_download_time) def makeMyOpener(head = { 'Connection': 'Keep-Alive', 'Accept': 'text/html, application/xhtml+xml, */*', 'Accept-Language': 'en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko'}): cj = http.cookiejar.CookieJar() opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) header = [] for key, value in head.items():  elem = (key, value)  header.append(elem) opener.addheaders = header return opener def saveFile(save_path,txts): f_obj = open(save_path,'w+') for item in txts:  f_obj.write(item+'/n') f_obj.close() #get_code_listcode_list='http://www.cishuge.com/read/0/771/'oper = makeMyOpener()uop = oper.open(code_list,timeout=1000)data = uop.read().decode('gbk','ignore')pattern = re.compile('<li><a href="(.*?)".*?>(.*?)</a></li>',re.S)items = re.findall(pattern,data)print ('获取列表完成')url_path='url_file.txt'url_r=open(url_path,'r')url_arr=url_r.readlines(100000)url_r.close()print (len(url_arr))url_file=open(url_path,'a')print ('获取已下载网址')for tmp in items: save_path = tmp[1].replace(' ','')+'.txt' url = code_list+tmp[0] if url+'/n' in url_arr:  continue print('写日志：'+url+'/n') url_file.write(url+'/n') opene = makeMyOpener() op1 = opene.open(url,timeout=1000) data = op1.read().decode('gbk','ignore') opene.close() pattern = re.compile('    (.*?)<br />',re.S) txts = re.findall(pattern,data) saveFile(save_path,txts) url_file.close()