Using Django with GAE Python 后台抓取多个网站的页面全文

application: l2zstory 
version: 1
runtime: python
api_version: 1

 

handlers:
   
- url: /images
  static_dir: l2zstory/templates/template2/images
- url: /css
  static_dir: l2zstory/templates/template2/css
- url: /js
  static_dir: l2zstory/templates/template2/js  
- url: /js
  static_dir: l2zstory/templates/template2/js
- url: /.*
  script: main.py

from django.conf.urls.defaults import *

 

# Uncomment the next two lines to enable the admin:
# from django.contrib import admin
# admin.autodiscover()

urlpatterns = patterns('',
    # Example:
    # (r'^l2zstory/', include('l2zstory.foo.urls')),

    # Uncomment the admin/doc line below and add 'django.contrib.admindocs'
    # to INSTALLED_APPS to enable admin documentation:
    # (r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    # (r'^admin/(.*)', admin.site.root),
    (r'^$','l2zstory.stories.views.L2ZStory'),
    (r'^YukiLife/','l2zstory.stories.views.YukiLife'),
     (r'^ZLife_Sina/','l2zstory.stories.views.ZLife_Sina'),
     (r'^ZLife/','l2zstory.stories.views.ZLife')
)

#from BeautifulSoup import BeautifulSoup 
from PyUtils import getAboutPage
from PyUtils import getPostInfos

 

def L2ZStory(request):
    url="feed://l2zstory.wordpress.com/feed/"
    about_url="http://l2zstory.wordpress.com/about/"
    blog_type="wordpress"
    htmlpages={}
    aboutContent=getAboutPage(about_url,blog_type)
    if aboutContent=="Not Found":
        aboutContent="We use this to tell those past stories..."
    htmlpages['about']={}
    htmlpages['about']['content']=aboutContent
    htmlpages['about']['title']="About This Story"
    htmlpages['about']['url']=about_url
    PostInfos=getPostInfos(url,blog_type,order_desc=True)
    return render_to_response('l2zstory.html',
{'PostInfos':PostInfos,
'htmlpages':htmlpages
})

import feedparser 
import urllib2
import re
from BeautifulSoup import BeautifulSoup
header={
'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:8.0.1) Gecko/20100101 Firefox/8.0.1',
}

timeoutMsg="""
The Robot cannot connect to the desired page due to either of these reasons:
1. Great Fire Wall
2. The Blog Site has block connections made by Robots.
"""

 

def getPageContent(url,blog_type): 
    try:
        req=urllib2.Request(url,None,header)
        response=urllib2.urlopen(req)
        html=response.read()
        html=BeautifulSoup(html).prettify()
        soup=BeautifulSoup(html)
        Content=""
        if blog_type=="wordpress":
            try:
                for Sharesection in soup.findAll('div',{'class':'sharedaddy sd-like-enabled sd-sharing-enabled'}):
                    Sharesection.extract()
                for item in soup.findAll('div',{'class':'post-content'}):
                    Content+=unicode(item)
            except:
                Content="No Post Content Found"
        elif blog_type=="sina":
            try:
                for item in soup.findAll('div',{'class':'articalContent  '}):
                    Content+=unicode(item)
            except:
                Content="No Post Content Found"

        ＃对于不同的网站类型 应用不同的过滤器

    except:
        Content=timeoutMsg
    return removeStyle(Content)

def removeStyle(Content):
    #add this to remove all the img tag : (<img +(/w=/".*/")>)|(</img>)|(src=/".*/")|
    patn=re.compile(r"(align=/".*/")|(id=/".*/")|(class=/"*/")|(style=/".*/")|(</font>)|(<font.*/">)|(<embed +(/w*=/".*/")>)|(</embed>)")
    replacepatn=""

    Content=re.sub(patn,replacepatn,Content)
    ＃运用正则表达式把抓取的内容中那些格式通通去掉 这样得到的文字比较纯粹
    return Content
   
def getPostInfos(url,blog_type,order_desc=False):
    feeds=feedparser.parse(url)
    PostInfos=[]
    if order_desc:
        items=feeds.entries[::-1]
    else:
        items=feeds.entries
    Cnt=0
    for  item in items:
        PostInfo={}
        PostInfo['title']=item.title
        PostInfo['author']=item.author
        PostInfo['date']=item.date
        PostInfo['link']=item.link
       
        if blog_type=="wordpress":
            Cnt+=1
            if Cnt<=8:
                PostInfo['description']=getPageContent(item.link,blog_type)
            else:
                PostInfo['description']=removeStyle(item.description)
        elif blog_type=="sina":
            PostInfo['description']=removeStyle(item.description)
           
       
        PostInfos.append(PostInfo)
       
    return PostInfos

<div class="page-header">
                              <a href="{{htmlpages.about.url}}" name="{{htmlpages.about.title}}"><h3>{{htmlpages.about.title}}</h3></a>
                             
                         </div>
                         <p>
                              {{htmlpages.about.content}}
                         </p>
                         {%for item in PostInfos%}
                         <div class="page-header">
                              <a href="{{item.link}}" name="{{item.title}}"><h3>{{item.title}}</h3></a>
                             
                         </div>
                         <p><i>author: {{item.author}}    date: {{item.date}}</i></p>
                         <p>{{item.description}}</p>
                         {%endfor%}
                    </div>