Python实现批量下载文件

#!/usr/bin/env python# -*- coding:utf-8 -*-from gevent import monkeymonkey.patch_all()from gevent.pool import Poolimport requestsimport sysimport osdef download(url): chrome = 'Mozilla/5.0 (X11; Linux i86_64) AppleWebKit/537.36 ' +  '(KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36' headers = {'User-Agent': chrome} filename = url.split('/')[-1].strip() r = requests.get(url.strip(), headers=headers, stream=True) with open(filename, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): if chunk:f.write(chunk)f.flush() print filename,"is ok"def removeLine(key, filename): os.system('sed -i /%s/d %s' % (key, filename))if __name__ =="__main__": if len(sys.argv) == 2: filename = sys.argv[1] f = open(filename,"r") p = Pool(4) for line in f.readlines(): if line: p.spawn(download, line.strip()) key = line.split('/')[-1].strip() removeLine(key, filename)f.close()p.join()else: print 'Usage: python %s urls.txt' % sys.argv[0]

from os.path import basenamefrom urlparse import urlsplitdef url2name(url):  return basename(urlsplit(url)[2])def download(url, localFileName = None):  localName = url2name(url)  req = urllib2.Request(url)  r = urllib2.urlopen(req)  if r.info().has_key('Content-Disposition'):    # If the response has Content-Disposition, we take file name from it    localName = r.info()['Content-Disposition'].split('filename=')[1]    if localName[0] == '"' or localName[0] == "'":      localName = localName[1:-1]  elif r.url != url:    # if we were redirected, the real file name we take from the final URL    localName = url2name(r.url)  if localFileName:    # we can force to save the file as specified name    localName = localFileName  f = open(localName, 'wb')  f.write(r.read())  f.close()download(r'你要下载的python文件的url地址')