python基于queue和threading实现多线程下载实例

  #download worker   queue_download = Queue.Queue(0)   DOWNLOAD_WORKERS = 20   for i in range(DOWNLOAD_WORKERS):     DownloadWorker(queue_download).start() #start a download worker        for md5 in MD5S:     queue_download.put(md5)   for i in range(DOWNLOAD_WORKERS):     queue_download.put(None) 

import threading import Queue import md5query import DOM import os,sys class DownloadWorker(threading.Thread):   """"""    def __init__(self, queue):     """Constructor"""     self.__queue = queue     threading.Thread.__init__(self)     def run(self):     while 1:       md5 = self.__queue.get()       if md5 is None:         break #reached end of queue       #this is a time-cost produce       self._down(md5)        print "task:", md5, "finished"    def _down(self, md5):     config = {       'input':sys.stdin,        'output':'./samples',        'location':'xxx',        'has-fn':False,        'options':{'connect.timeout':60, 'timeout':3600},        'log':file('logs.txt', 'w'),      }     print 'download %s...' % (md5)     try:       data = downloadproc(config['location'], config['options'])#我的下载过程       if data:         dom, fileData = md5query.splited(data)         filename = md5         if config['has-fn']:           filename = '%s_%s' % (md5, dom.nodeValue2('xxxxxxx', '').encode('utf-8'))#这是我的下载的方法         f = file(os.path.join(config['output'], filename), 'w')         f.write(fileData)         f.close()          print '%s/tok' % (md5)       else:         print>>config['log'], '%s/t%s' % (md5, 'failed')     except Exception, e:       print>>config['log'], '%s/t%s' % (md5, str(e))