python+mongodb数据抓取详细介绍

def step():  try:    headers = {      。。。。。      }    r = requests.get(url,headers,timeout=30)    html = r.content    soup = BeautifulSoup(html,"lxml")    url = soup.find_all(正则表达式)    for i in url:      url2 = i.find_all('a')      for j in url2:         step1url =url + j['href']         print step1url         step2(step1url)  except Exception,e:    print e

def step2(step1url):  try:    headers = {      。。。。      }    r = requests.get(step1url,headers,timeout=30)    html = r.content    soup = BeautifulSoup(html,"lxml")    a = soup.find('div',id='divTbl')    if a:      url = soup.find_all('td',class_='S-ITabs')      for i in url:        classifyurl = i.find_all('a')        for j in classifyurl:           step2url = url + j['href']           #print step2url           step3(step2url)    else:      postdata(step1url)

def producturl(url):  try:    p1url = doc.xpath(正则表达式)    for i in xrange(1,len(p1url) + 1):      p2url = doc.xpath(正则表达式)      if len(p2url) > 0:        producturl = url + p2url[0].get('href')        count = db[table].find({'url':producturl}).count()        if count <= 0:            sn = getNewsn()            db[table].insert({"sn":sn,"url":producturl})            print str(sn) + 'inserted successfully'        else:            'url exist'  except Exception,e:    print e

def parser(sn,url):  try:    headers = {      。。。。。。      }    r = requests.get(url, headers=headers,timeout=30)    html = r.content    soup = BeautifulSoup(html,"lxml")    dt = {}    #partno    a = soup.find("meta",itemprop="mpn")    if a:      dt['partno'] = a['content']    #manufacturer    b = soup.find("meta",itemprop="manufacturer")    if b:      dt['manufacturer'] = b['content']    #description    c = soup.find("span",itemprop="description")    if c:      dt['description'] = c.get_text().strip()    #price    price = soup.find("table",class_="table table-condensed occalc_pa_table")    if price:      cost = {}      for i in price.find_all('tr'):        if len(i) > 1:          td = i.find_all('td')          key=td[0].get_text().strip().replace(',','')          val=td[1].get_text().replace(u'/u20ac','').strip()          if key and val:            cost[key] = val      if cost:        dt['cost'] = cost        dt['currency'] = 'EUR'    #quantity    d = soup.find("input",id="ItemQuantity")    if d:      dt['quantity'] = d['value']    #specs    e = soup.find("div",class_="row parameter-container")    if e:      key1 = []      val1= []      for k in e.find_all('dt'):        key = k.get_text().strip().strip('.')        if key:          key1.append(key)      for i in e.find_all('dd'):        val = i.get_text().strip()        if val:          val1.append(val)      specs = dict(zip(key1,val1))    if specs:      dt['specs'] = specs      print dt          if dt:      db[table].update({'sn':sn},{'$set':dt})      print str(sn) + ' insert successfully'      time.sleep(3)    else:      error(str(sn) + '/t' + url)  except Exception,e:    error(str(sn) + '/t' + url)    print "Don't data!"