# -*- coding: utf-8 -*-import requestsfrom bs4 import BeautifulSoupfrom requests.exceptions import RequestExceptionimport timeimport randomimport MySQLdbimport threadingimport socketimport math socket.setdefaulttimeout(60)#这里对整个socket层设置超时时间。后续文件中如果再使用到socket，不必再设置 glock = threading.Lock() #定义全局锁 CATEGORY_URL= ['http://www.we123.com/gzh/onclick/'] #获取地区分类链接all_url = [] #ALL_URLS = [] #所有详细页面链接proxy_list = [] #IP池URL = 'http://www.we123.com'PAGE_URL = [] #所有分页链接 #获取Ip池def get_ip(): headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'} url = 'http://http-webapi.zhimaruanjian.com'#可以使用芝麻代理，好用稳定还不贵 resp = requests.get(url,headers=headers) obj = resp.json() #获取json ip池对象 for ip in obj: arr = 'http://' + str(ip['ip']) + ':' + str(ip['port']) proxy_list.append(arr) #获取页面源码函数def get_html(url): # headers = {} user_agent_list = [ 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3538.400 QQBrowser/9.6.12501.400', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0' ] # user_agent = random.choice(user_agent_list) headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3538.400 QQBrowser/9.6.12501.400' } # 代理，免费的代理只能维持一会可能就没用了，自行更换 # proxy_list = [ # "http://27.192.185.62:3252", # ] # proxy_ip = random.choice(proxy_list) # proxies = {'http': proxy_ip} # print(str(url)) try: resp = requests.get(url,headers=headers) # print("72行：status_code = " + str(resp.status_code)) # print(type(resp.text)) # print(resp.url) # 请求的url if resp.status_code == 200: return resp elif resp.status_code == 404: return resp elif resp.status_code == 500: return resp return resp except RuntimeError: print("超时") return "error" except ConnectionError: print("连接超时") return "error" except RequestException: print("http请求父类错误") with open('url_exception.txt','a+', encoding='utf-8') as f: f.write(str(url)) f.write('/n') return "error" #获取区域分类链接def get_categoty_url(): url = 'http://www.we123.com/gzh/onclick/' resp = get_html(url) soup = BeautifulSoup(resp.text,'lxml') html = soup.select('div.div-subs2 > div.divst-content > div.divst-subs > li > a') # 获取区域分类链接 for i in html: city = i['href'].split("/")[-1] if (city == '海外' or city == '台湾' or city == '澳门'): continue url = URL + i['href'] CATEGORY_URL.append(url) print(CATEGORY_URL) #获取每个区域下所有分页链接def get_page_url(url): city = url.split('/')[-1] html = get_html(url) if html == "error": print("98行：connect url error") time.sleep(random.randint(10,20)) return "error" soup = BeautifulSoup(html.text,'lxml') #获取总条数 all_nums = soup.select("div.page > a > b") if len(all_nums) == 0: return "error" else: all_nums = soup.select("div.page > a > b")[0].get_text() #获取总分页数 all_pages = math.ceil((int(all_nums) / 30)) #获取所有分页链接 all_page_url = [] for i in range(0,int(all_pages)): page_url = 'http://www.we123.com/e/action/ListInfo.php?page=' + str(i) + '&classid=45&line=30&tempid=10&orderby=onclick&myorder=0&totalnum=' + str(all_nums) all_page_url.append(page_url) return all_page_url # 获取所有详细页面链接def get_page_urls(): global PAGE_URL c_url = CATEGORY_URL.pop() print('121 行：请求链接' + c_url) PAGE_URL = get_page_url(c_url) #获取每个区域下面的所有分页链接 # 获取所有详细页面链接def get_info_urls(): while True: global PAGE_URL #设置全局变量 glock.acquire() #加锁 if len(PAGE_URL) == 0: glock.release() #解锁 print('131 行：CATEGORY_URL 为空') break else: p_url = PAGE_URL.pop() print('135 行：请求链接' + p_url) glock.release() #解锁 glock.acquire() #加锁 html = get_html(p_url) if html == "error": print("141行：connect url error") time.sleep(2) return soup = BeautifulSoup(html.text,'lxml') info_urls = soup.select('div.gzhRight > div.gzh_list > ul > li > a') for x in info_urls: i_url = URL + x['href'] ALL_URLS.append(i_url) print("库存链接共:" + str(len(ALL_URLS))) glock.release() #解锁#获取每一页需要的数据def get_data(): while True: global ALL_URLS #设置全局变量 glock.acquire() #加锁 print("当前库存："+str(len(ALL_URLS))) if len(ALL_URLS) == 0: glock.release() #解锁 print('159 行 ：ALL_URLS 为空') break else: url = ALL_URLS.pop() print("开始抓取数据：" + url) glock.release() #解锁 time.sleep(1) #睡眠1秒钟 html = get_html(url) if html == "error": print("168行：connect url error") time.sleep(random.randint(2, 4)) return html.encoding='utf-8' #显式地指定网页编码，一般情况可以不用 soup = BeautifulSoup(html.text,'lxml') #公众号名称 names = soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.gzhtop_logo > h1') #微信号id accounts = [] accounts.append(soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.gzhtop_logo > p')[0]) #微信头像 imgs = soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.gzhtop_logo > img') #公众号二维码 QR_codes= soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_right > img') #介绍 descs = soup.select('div.artcleLeft > div.xcxnry > div.xcxinfo') #公众号分类 categorys = [] category = '' cate = soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.xcx_p > span > a') if not len(cate) == 0: category = cate[0].get_text() else: category = '综合' glock.acquire() #加锁 for name,account,img,QR_code,desc in zip(names,accounts,imgs,QR_codes,descs): data = { 'name':name.get_text(), 'category':category, 'account':account.get_text().split("：")[-1], 'img':img['src'], 'QR_code':QR_code['src'], 'desc':desc.get_text() } add_data(data,url) glock.release() #解锁#添加数据def add_data(data,url): con = MySQLdb.connect('127.0.0.1','root','root','test',charset="utf8",use_unicode=True) cursor = con.cursor() # exit() insert_sql = """ insert ignore into weixin5(w_name,category,account,img,QR_code,introduce) VALUES (%s,%s,%s,%s,%s,%s) """ print('212行 ：' + data['name'] + '_' + data['account'] + '添加成功！-' + url) try: cursor.execute(insert_sql,(data['name'],data['category'],data['account'],data['img'],data['QR_code'],str(data['desc']))) con.commit() except: ALL_URLS.insert(0,url) print("218行：" + URL + '插入失败') con.rollback() con.close() # 将时间字符串转化为时间戳def time_to(dt): timeArray = time.strptime(dt, "%Y年%m月%d日") timestamp = int(time.mktime(timeArray)) return timestamp #启动多线程爬取def main(): for x in range(3): th = threading.Thread(target=get_info_urls) th.start() # get_info_urls() time.sleep(3) for x in range(5): th = threading.Thread(target=get_data) th.start() if __name__ == '__main__': # 计时 t1 = time.time() # 调用函数 get_ip() #获取ip池 get_page_urls() time.sleep(2) # get_categoty_url() main() print(time.time() - t1) 
