python实现多线程网页下载器_Python

本文为大家分享了python实现的一个多线程网页下载器，供大家参考，具体内容如下

这是一个有着真实需求的实现，我的用途是拿它来通过 HTTP 方式向服务器提交游戏数据。把它放上来也是想大家帮忙挑刺，找找 bug，让它工作得更好。

keywords:python,http,multi-threads,thread,threading,httplib,urllib,urllib2,Queue,http pool,httppool

废话少说，上源码：

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

									# -*- coding:utf-8 -*- 

									import urllib, httplib 

									import thread 

									import time 

									from Queue import Queue, Empty, Full 

									HEADERS = {"Content-type": "application/x-www-form-urlencoded", 

									            'Accept-Language':'zh-cn', 

									            'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0;Windows NT 5.0)', 

									            "Accept": "text/plain"} 

									UNEXPECTED_ERROR = -1

									POST = 'POST'

									GET = 'GET'

									def base_log(msg): 

									  print msg 

									def base_fail_op(task, status, log): 

									  log('fail op. task = %s, status = %d'%(str(task), status)) 

									def get_remote_data(tasks, results, fail_op = base_fail_op, log = base_log): 

									  while True: 

									    task = tasks.get() 

									    try: 

									      tid = task['id'] 

									      hpt = task['conn_args'] # hpt <= host:port, timeout 

									    except KeyError, e: 

									      log(str(e)) 

									      continue

									    log('thread_%s doing task %d'%(thread.get_ident(), tid)) 

									    #log('hpt = ' + str(hpt)) 

									    conn = httplib.HTTPConnection(**hpt) 

									    try: 

									      params = task['params'] 

									    except KeyError, e: 

									      params = {} 

									    params = urllib.urlencode(params) 

									    #log('params = ' + params) 

									    try: 

									      method = task['method'] 

									    except KeyError: 

									      method = 'GET'

									    #log('method = ' + method) 

									    try: 

									      url = task['url'] 

									    except KeyError: 

									      url = '/'

									    #log('url = ' + url) 

									    headers = HEADERS 

									    try: 

									      tmp = task['headers'] 

									    except KeyError, e: 

									      tmp = {} 

									    headers.update(tmp) 

									    #log('headers = ' + str(headers)) 

									    headers['Content-Length'] = len(params) 

									    try: 

									      if method == POST: 

									        conn.request(method, url, params, headers) 

									      else: 

									        conn.request(method, url + params) 

									      response = conn.getresponse() 

									    except Exception, e: 

									      log('request failed. method = %s, url = %s, params = %s headers = %s'%( 

									            method, url, params, headers)) 

									      log(str(e)) 

									      fail_op(task, UNEXPECTED_ERROR, log) 

									      continue

									    if response.status != httplib.OK: 

									      fail_op(task, response.status, log) 

									      continue

									    data = response.read() 

									    results.put((tid, data), True) 

									class HttpPool(object): 

									  def __init__(self, threads_count, fail_op, log): 

									    self._tasks = Queue() 

									    self._results = Queue() 

									    for i in xrange(threads_count): 

									      thread.start_new_thread(get_remote_data,  

									                              (self._tasks, self._results, fail_op, log)) 

									  def add_task(self, tid, host, url, params, headers = {}, method = 'GET', timeout = None): 

									    task = { 

									      'id' : tid, 

									      'conn_args' : {'host' : host} if timeout is None else {'host' : host, 'timeout' : timeout}, 

									      'headers' : headers, 

									      'url' : url, 

									      'params' : params, 

									      'method' : method, 

									      } 

									    try: 

									      self._tasks.put_nowait(task) 

									    except Full: 

									      return False

									    return True

									  def get_results(self): 

									    results = [] 

									    while True: 

									      try: 

									        res = self._results.get_nowait() 

									      except Empty: 

									        break

									      results.append(res) 

									    return results 

									def test_google(task_count, threads_count): 

									  hp = HttpPool(threads_count, base_fail_op, base_log) 

									  for i in xrange(task_count): 

									    if hp.add_task(i, 

									        'www.google.cn', 

									        '/search?', 

									        {'q' : 'lai'}, 

									#        method = 'POST' 

									        ): 

									      print 'add task successed.'

									  while True: 

									    results = hp.get_results() 

									    if not results: 

									      time.sleep(1.0 * random.random()) 

									    for i in results: 

									      print i[0], len(i[1]) 

									#      print unicode(i[1], 'gb18030') 

									if __name__ == '__main__': 

									  import sys, random 

									  task_count, threads_count = int(sys.argv[1]), int(sys.argv[2]) 

									  test_google(task_count, threads_count)