#encoding:utf8import refrom selenium import webdriverfrom selenium.common.exceptions import TimeoutExceptionfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECfrom bs4 import BeautifulSoupfrom config import *import pymongoclient=pymongo.MongoClient(MONGODB_URL)db=client[MONGODB_DB]##这里使用PhantomJS，并配置了一些参数browser=webdriver.PhantomJS(service_args=SERVICE_ArGS)##窗口的大小，不设置的话，默认太小，会有问题browser.set_window_size(1400,900)wait=WebDriverWait(browser, 10)def search(): print('正在搜索')##容易出现超时的错误 try:##等待这两个模块都加载好 browser.get("https://www.taobao.com") input = wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, '#q')) ) submit=wait.until( EC.element_to_be_clickable((By.CSS_SELECTOR, '#J_TSearchForm > div.search-button > button')) ) ######这块python2搞得鬼 #input.send_keys('/u7f8e/u98df'.decode("unicode-escape")) input.send_keys(KEYWORD.decode("unicode-escape")) submit.click() total = wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.total')) ) get_product() return total.text except TimeoutException: return search()def next_page(page_number): print('翻页'+str(page_number)) try: input = wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > input')) ) submit=wait.until( EC.element_to_be_clickable((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit')) ) input.clear() input.send_keys(page_number) submit.click()##判断是否翻页成功 高亮的是不是输入的值，直接加在后面即可 wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR,'#mainsrp-pager > div > div > div > ul > li.item.active > span'),str(page_number))) get_product() except TimeoutException: return next_page(page_number)#获取产品信息def get_product(): products = wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-itemlist .m-itemlist .items')) )##拿到网页 html = browser.page_source soup = BeautifulSoup(html,'lxml') items = soup.select('#mainsrp-itemlist .m-itemlist .items .item.J_MouserOnverReq')# print('*************************到此*************') for item in items: img = item.select('.J_ItemPic.img')[0].get('src') price = item.select('.price.g_price.g_price-highlight > strong')[0].get_text() deal = item.select('.deal-cnt')[0].get_text() shop = item.select('.row.row-3.g-clearfix > .shop > a > span:nth-of-type(2)')[0].get_text() location = item.select('.location')[0].get_text() product={ 'img':img, 'price':price, 'deal':deal, 'title':title, 'shop':shop, 'location':location }#打印一下 import json j = json.dumps(product) dict2 = j.decode("unicode-escape") print dict2 save_to_mongo(product)def save_to_mongo(product): try: if db[MONGODB_TABLE].insert(product): print('存储到mongodb成功'+str(product)) except Exception: print("存储到mongodb失败"+str(product))def main(): try: total=search() ##搜寻 re正则表达式 s=re.compile('(/d+)') total=int(s.search(total).group(1)) for i in range(2,total+1): next_page(i) except Exception: print('出错') finally: browser.close()if __name__ == '__main__': main() 
