使用Python编写爬虫的基本模块及框架使用指南

#!coding=utf-8import urllibimport urllib2 url = 'http://www.baidu.com/'# 获取请求request = urllib2.Request(url)try:  # 根据request，得到返回response  response = urllib2.urlopen(request)except urllib2.HTTPError, e:  if hasattr(e, 'reason'):    print e.reason# 读取response的bodyhtml = response.read()# 读取response的headersheaders = response.info()

#!coding=utf-8import urllib2import urllib post_url = '' post_data = urllib.urlencode({  'username': 'username',  'password': 'password',}) post_headers = {  'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:31.0) Gecko/20100101 Firefox/31.0',} request = urllib2.Request(  url=post_url,  data=post_data,  headers=post_headers,) response = urllib2.urlopen(request) html = response.read()

#!coding=utf-8 import urllib2import re page_num = 1url = 'http://tieba.baidu.com/p/3238280985?see_lz=1&pn='+str(page_num)myPage = urllib2.urlopen(url).read().decode('gbk') myRe = re.compile(r'class="d_post_content j_d_post_content ">(.*?)</div>', re.DOTALL)items = myRe.findall(myPage) f = open('baidu.txt', 'a+') import sysreload(sys)sys.setdefaultencoding('utf-8') i = 0texts = []for item in items:  i += 1  print i  text = item.replace('<br>', '')  text.replace('/n', '').replace(' ', '') + '/n'  print text  f.write(text) f.close()

#coding:utf-8'''  模拟登陆163邮箱并下载邮件内容 '''import urllibimport urllib2import cookielibimport reimport timeimport json class Email163:  header = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}  user = ''  cookie = None  sid = None  mailBaseUrl='http://twebmail.mail.163.com'   def __init__(self):    self.cookie = cookielib.CookieJar()    cookiePro = urllib2.HTTPCookieProcessor(self.cookie)    urllib2.install_opener(urllib2.build_opener(cookiePro))   def login(self,user,pwd):    '''      登录    '''    postdata = urllib.urlencode({        'username':user,        'password':pwd,        'type':1      })    #注意版本不同，登录URL也不同    req = urllib2.Request(        url='https://ssl.mail.163.com/entry/coremail/fcg/ntesdoor2?funcid=loginone&language=-1&passtype=1&iframe=1&product=mail163&from=web&df=email163&race=-2_45_-2_hz&module=&uid='+user+'&style=10&net=t&skinid=null',        data=postdata,        headers=self.header,      )    res = str(urllib2.urlopen(req).read())    #print res    patt = re.compile('sid=([^"]+)',re.I)    patt = patt.search(res)     uname = user.split('@')[0]    self.user = user    if patt:      self.sid = patt.group(1).strip()      #print self.sid      print '%s Login Successful.....'%(uname)    else:      print '%s Login failed....'%(uname)    def getInBox(self):    '''      获取邮箱列表    '''    print '/nGet mail lists...../n'    sid = self.sid    url = self.mailBaseUrl+'/jy3/list/list.do?sid='+sid+'&fid=1&fr=folder'    res = urllib2.urlopen(url).read()    #获取邮件列表    mailList = []    patt = re.compile('<div/s+class="tdLike Ibx_Td_From"[^>]+>.*?href="([^"]+)"[^>]+>(.*?)<//a>.*?<div/s+class="tdLike Ibx_Td_Subject"[^>]+>.*?href="[^>]+>(.*?)<//a>',re.I|re.S)    patt = patt.findall(res)    if patt==None:      return mailList     for i in patt:      line = {          'from':i[1].decode('utf8'),           'url':self.mailBaseUrl+i[0],           'subject':i[2].decode('utf8')           }      mailList.append(line)     return mailList    def getMailMsg(self,url):    '''      下载邮件内容    '''    content=''    print '/n Download.....%s/n'%(url)    res = urllib2.urlopen(url).read()     patt = re.compile('contentURL:"([^"]+)"',re.I)    patt = patt.search(res)    if patt==None:      return content    url = '%s%s'%(self.mailBaseUrl,patt.group(1))    time.sleep(1)    res = urllib2.urlopen(url).read()    Djson = json.JSONDecoder(encoding='utf8')    jsonRes = Djson.decode(res)    if 'resultVar' in jsonRes:      content = Djson.decode(res)['resultVar']    time.sleep(3)    return content  '''  Demon'''#初始化mail163 = Email163()#登录mail163.login('lpe234@163.com','944898186')time.sleep(2) #获取收件箱elist = mail163.getInBox() #获取邮件内容for i in elist:  print '主题：%s  来自：%s 内容：/n%s'%(i['subject'].encode('utf8'),i['from'].encode('utf8'),mail163.getMailMsg(i['url']).encode('utf8'))

#1 cookie的处理 import urllib2, cookielibcookie_support= urllib2.HTTPCookieProcessor(cookielib.CookieJar())opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)urllib2.install_opener(opener)content = urllib2.urlopen('http://XXXX').read() #2 用代理和cookie opener = urllib2.build_opener(proxy_support, cookie_support, urllib2.HTTPHandler) #3 表单的处理 import urllibpostdata=urllib.urlencode({  'username':'XXXXX',  'password':'XXXXX',  'continueURI':'http://www.verycd.com/',  'fk':fk,  'login_submit':'登录'}) req = urllib2.Request(  url = 'http://secure.verycd.com/signin/*/http://www.verycd.com/',  data = postdata)result = urllib2.urlopen(req).read() #4 伪装成浏览器访问 headers = {  'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}req = urllib2.Request(  url = 'http://secure.verycd.com/signin/*/http://www.verycd.com/',  data = postdata,  headers = headers) #5 反”反盗链” headers = {  'Referer':'http://www.cnbeta.com/articles'}

from threading import Threadfrom Queue import Queuefrom time import sleep#q是任务队列#NUM是并发线程总数#JOBS是有多少任务q = Queue()NUM = 2JOBS = 10#具体的处理函数，负责处理单个任务def do_somthing_using(arguments):  print arguments#这个是工作进程，负责不断从队列取数据并处理def working():  while True:    arguments = q.get()    do_somthing_using(arguments)    sleep(1)    q.task_done()#fork NUM个线程等待队列for i in range(NUM):  t = Thread(target=working)  t.setDaemon(True)  t.start()#把JOBS排入队列for i in range(JOBS):  q.put(i)#等待所有JOBS完成q.join()

# 使用 scrapy startproject scrapy_test├── scrapy_test│  ├── scrapy.cfg│  └── scrapy_test│    ├── __init__.py│    ├── items.py│    ├── pipelines.py│    ├── settings.py│    └── spiders│      ├── __init__.py# 进行创建 scrapy 项目

# python-dev 包的安装apt-get install python-dev # twisted, w3lib, six, queuelib, cssselect， libxslt pip install w3libpip install twistedpip install lxmlapt-get install libxml2-dev libxslt-dev apt-get install python-lxmlpip install cssselect pip install pyOpenSSL sudo pip install service_identity # 安装好之后，便可使用 scrapy startproject test 进行创建项目

dizzy@dizzy-pc:~/Python/spit$ scrapy startproject itzhaopinNew Scrapy project 'itzhaopin' created in:  /home/dizzy/Python/spit/itzhaopin You can start your first spider with:  cd itzhaopin  scrapy genspider example example.comdizzy@dizzy-pc:~/Python/spit$  dizzy@dizzy-pc:~/Python/spit$ cd itzhaopindizzy@dizzy-pc:~/Python/spit/itzhaopin$ tree.├── itzhaopin│  ├── __init__.py│  ├── items.py│  ├── pipelines.py│  ├── settings.py│  └── spiders│    └── __init__.py└── scrapy.cfg # scrapy.cfg: 项http://my.oschina.net/lpe234/admin/new-blog目配置文件# items.py: 需要提取的数据结构定义文件# pipelines.py:管道定义，用来对items里面提取的数据做进一步处理，如保存等# settings.py: 爬虫配置文件# spiders: 放置spider的目录

from scrapy.item import Item, Field# 定义我们要抓取的数据class TencentItem(Item):  name = Field() # 职位名称  catalog = Field() # 职位类别  workLocation = Field() # 工作地点  recruitNumber = Field() # 招聘人数  detailLink = Field() # 职位详情链接  publishTime = Field() # 发布时间

#coding=utf-8 from scrapy.spider import BaseSpider  class DmozSpider(BaseSpider):  name = 'dmoz'  allowed_domains = ['dmoz.org']  start_urls = [    'http://www.dmoz.org/Computers/Programming/Languages/Python/Books/',    'http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/'  ]   def parse(self, response):    filename = response.url.split('/')[-2]    open(filename, 'wb').write(response.info)