Rev 6430: (gz) Remove tools/http_client.py (Martin Packman) in file:///srv/pqm.bazaar-vcs.org/archives/thelove/bzr/%2Btrunk/

Patch Queue Manager pqm at pqm.ubuntu.com
Thu Jan 5 15:04:41 UTC 2012


At file:///srv/pqm.bazaar-vcs.org/archives/thelove/bzr/%2Btrunk/

------------------------------------------------------------
revno: 6430 [merge]
revision-id: pqm at pqm.ubuntu.com-20120105150440-7yb2l2c4dnnhx9y5
parent: pqm at pqm.ubuntu.com-20120105143743-ysvb9ea44dklacs3
parent: martin.packman at canonical.com-20120105122131-3r85iyoyc30q13jz
committer: Patch Queue Manager <pqm at pqm.ubuntu.com>
branch nick: +trunk
timestamp: Thu 2012-01-05 15:04:40 +0000
message:
  (gz) Remove tools/http_client.py (Martin Packman)
removed:
  tools/http_client.py           http_client.py-20050819224207-b29be8324c55b359
=== removed file 'tools/http_client.py'
--- a/tools/http_client.py	2006-09-08 18:46:29 +0000
+++ b/tools/http_client.py	1970-01-01 00:00:00 +0000
@@ -1,501 +0,0 @@
-#! /usr/bin/python
-
-# $Id: http_client.py 271 2004-10-09 10:50:59Z fredrik $
-# a simple asynchronous http client (based on SimpleAsyncHTTP.py from
-# "Python Standard Library" by Fredrik Lundh, O'Reilly 2001)
-#
-# HTTP/1.1 and GZIP support added in January 2003 by Fredrik Lundh.
-#
-# changes:
-# 2004-08-26 fl   unified http callback
-# 2004-10-09 fl   factored out gzip_consumer support
-# 2005-07-08 mbp  experimental support for keepalive connections
-#
-# Copyright (c) 2001-2004 by Fredrik Lundh.  All rights reserved.
-#
-
-
-
-"""async/pipelined http client
-
-Use
-===
-
-Users of this library pass in URLs they want to see, and consumer
-objects that will receive the results at some point in the future.
-Any number of requests may be queued up, and more may be added while
-the download is in progress.
-
-Requests can be both superscalar and superpipelined.  That is to say,
-for each server there can be multiple sockets open, and each socket
-may have more than one request in flight.
-
-Design
-======
-
-There is a single DownloadManager, and a connection object for each
-open socket.
-
-Request/consumer pairs are maintained in queues.  Each connection has
-a list of transmitted requests whose response has not yet been
-received.  There is also a per-server list of requests that have not
-yet been submitted.
-
-When a connection is ready to transmit a new request, it takes one
-from the unsubmitted list, sends the request, and adds the request to
-its unfulfilled list.  This should happen when the connection has
-space for more transmissions or when a new request is added by the
-user.  If the connection terminates with unfulfilled requests they are
-put back onto the unsubmitted list, to be retried elsewhere.
-
-Because responses come back precisely in order, the connection always
-knows what it should expect next: the response for the next
-unfulfilled request.
-"""
-
-# Note that (as of ubuntu python 2.4.1) every socket.connect() call
-# with a hostname does a remote DNS resolution, which is pretty sucky.
-# Shouldn't there be a cache in glibc?  We should probably cache the
-# address in, say, the DownloadManager.
-
-# TODO: A default consumer operation that writes the received data
-# into a file; by default the file is named the same as the last
-# component of the URL.
-
-# TODO: A utility function that is given a list of URLs, and downloads
-# them all parallel/pipelined.  If any fail, it raises an exception
-# (and discards the rest), or perhaps can be told to continue anyhow.
-# The content is written into temporary files.  It returns a list of
-# readable file objects.
-
-# TODO: If we try pipelined or keepalive and the connection drop out
-# then retry the request on a new connection; eventually we should perhaps
-# learn that a given host or network just won't allow keepalive.
-
-
-import asyncore
-import socket, string, time, sys
-import StringIO
-import mimetools, urlparse, urllib
-import logging
-
-logging.basicConfig(level=logging.DEBUG,
-                    format='%(asctime)s %(levelname)s %(message)s',
-                    filename='/tmp/http_client.log',
-                    filemode='w')
-
-logger = logging.getLogger('bzr.http_client')
-debug = logger.debug
-info = logger.info
-error = logger.error
-
-
-##
-# Close connection.   Request handlers can raise this exception to
-# indicate that the connection should be closed.
-
-class CloseConnection(Exception):
-    pass
-
-##
-# Redirect connection.  Request handlers can raise this exception to
-# indicate that the a new request should be issued.
-
-class Redirect(CloseConnection):
-    def __init__(self, location):
-        self.location = location
-
-
-class DownloadManager(object):
-    """Handles pipelined/overlapped downloads.
-
-    Pass in a series of URLs with handlers to receive the response.
-    This object will spread the requests over however many sockets
-    seem useful.
-
-    queued_requests
-        Requests not assigned to any channel
-
-    running_requests
-        Currently assigned to a channel
-    """
-    def __init__(self):
-        self.queued_requests = []
-        # self.channel = HttpChannel('localhost', 8000, self)
-        self.channels = []
-        self.try_pipelined = False
-        self.try_keepalive = False
-        self.max_channels = 5
-
-
-    def enqueue(self, url, consumer):
-        self.queued_requests.append((url, consumer))
-        self._wake_up_channel()
-
-
-    def _channel_closed(self, channel):
-        """Called by the channel when its socket closes.
-        """
-        self.channels.remove(channel)
-        if self.queued_requests:
-            # might recreate one
-            self._wake_up_channel()
-
-
-    def _make_channel(self):
-        # proxy2 203.17.154.69
-        # return HttpChannel('82.211.81.161', 80, self)         # bazaar-ng.org 
-        # return HttpChannel('203.17.154.69', 8080, self)
-        return HttpChannel('127.0.0.1', 8000, self)  # forwarded
-            
-
-    def _wake_up_channel(self):
-        """Try to wake up one channel to send the newly-added request.
-
-        There may be more than one request pending, and this may cause
-        more than one channel to take requests.  That's OK; some of
-        them may be frustrated.
-        """
-        from random import shuffle, choice
-        
-        # first, wake up any idle channels
-        done = False
-        for ch in self.channels:
-            if not ch.sent_requests:
-                ch.take_one()
-                done = True
-        if done:
-            debug("woke existing idle channel(s)")
-            return
-
-        if len(self.channels) < self.max_channels:
-            newch = self._make_channel()
-            self.channels.append(newch)
-            newch.take_one()
-            debug("created new channel")
-            return
-
-        if self.try_pipelined:
-            # ask existing channels to take it
-            debug("woke busy channel")
-            choice(self.channels).take_one()
-
-
-        # debug("request postponed until a channel's idle")
-        
-
-
-
-    def run(self):
-        """Run until all outstanding requests have been served."""
-        #while self.running_requests or self.queued_requests \
-        #          or not self.channel.is_idle():
-        #    asyncore.loop(count=1)
-        asyncore.loop()
-
-
-
-class Response(object):
-    """Holds in-flight response."""
-
-
-
-def _parse_response_http10(header):
-    from cStringIO import StringIO
-
-    fp = StringIO(header)
-    r = Response()
-
-    r.status = fp.readline().split(" ", 2)
-    r.headers = mimetools.Message(fp)
-
-    # we can only(?) expect to do keepalive if we got either a 
-    # content-length or chunked encoding; otherwise there's no way to know
-    # when the content ends apart from through the connection close
-    r.content_type = r.headers.get("content-type")
-    try:
-        r.content_length = int(r.headers.get("content-length"))
-    except (ValueError, TypeError):
-        r.content_length = None
-    debug("seen content length of %r" % r.content_length)
-
-    r.transfer_encoding = r.headers.get("transfer-encoding")
-    r.content_encoding = r.headers.get("content-encoding")
-    r.connection_reply = r.headers.get("connection")
-
-    # TODO: pass status code to consumer?
-
-    if r.transfer_encoding:
-        raise NotImplementedError()
-
-    if r.transfer_encoding:
-        raise NotImplementedError()
-
-    if int(r.status[1]) != 200:
-        debug("can't handle response status %r" % r.status)
-        raise NotImplementedError()
-
-    if r.content_length is None:
-        raise NotImplementedError()
-
-    if r.content_length == 0:
-        raise NotImplementedError()
-
-    r.content_remaining = r.content_length                
-
-    return r
-
-
-    
-    
-        
-
-
-class HttpChannel(asyncore.dispatcher_with_send):
-    """One http socket, pipelining if possible."""
-    # asynchronous http client
-
-    user_agent = "http_client.py 1.3ka (based on effbot)"
-
-    proxies = urllib.getproxies()
-
-    def __init__(self, ip_host, ip_port, manager):
-        asyncore.dispatcher_with_send.__init__(self)
-        self.manager = manager
-
-        # if a response header has been seen, this holds it
-        self.response = None
-        
-        self.data = ""
-
-        self.chunk_size = None
-
-        self.timestamp = time.time()
-
-        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
-        debug('connecting...')
-        self.connect((ip_host, ip_port))
-
-        # sent_requests holds (url, consumer) 
-        self.sent_requests = []
-
-        self._outbuf = ''
-
-
-    def __repr__(self):
-        return 'HttpChannel(local_port=%r)' % (self.getsockname(),)
-
-
-    def is_idle(self):
-        return (not self.sent_requests)
-
-
-    def handle_connect(self):
-        debug("connected")
-        self.take_one()
-
-
-    def take_one(self):
-        """Accept one request from the manager if possible."""
-        if self.manager.try_pipelined:
-            if len(self.sent_requests) > 4:
-                return
-        else:
-            if len(self.sent_requests) > 0:
-                return 
-        
-        try:
-            url, consumer = self.manager.queued_requests.pop(0)
-            debug('request accepted by channel')
-        except IndexError:
-            return
-        
-        # TODO: If there are too many already in flight, don't take one.
-        # TODO: If the socket's not writable (tx buffer full), don't take.
-        self._push_request_http10(url, consumer)
-
-
-
-    def _push_request_http10(self, url, consumer):
-        """Send a request, and add it to the outstanding queue."""
-        # TODO: check the url requested is appropriate for this connection
-
-        # TODO: If there are too many requests outstanding or (less likely) the 
-        # connection fails, queue it for later use.
-
-        # TODO: Keep track of requests that have been sent but not yet fulfilled,
-        # because we might need to retransmit them if the connection fails. (Or
-        # should the caller do that?)
-
-        request = self._form_request_http10(url)
-        debug('send request for %s from %r' % (url, self))
-
-        # dispatcher_with_send handles buffering the data until it can
-        # be written, and hooks handle_write.
-
-        self.send(request)
-
-        self.sent_requests.append((url, consumer))
-
-
-    def _form_request_http10(self, url):
-        # TODO: get right vhost name
-        request = [
-            "GET %s HTTP/1.0" % (url),
-            "Host: www.bazaar-ng.org",
-            ]
-
-        if self.manager.try_keepalive or self.manager.try_pipelined:
-            request.extend([
-                "Keep-Alive: 60", 
-                "Connection: keep-alive",
-                ])
-
-        # make sure to include a user agent
-        for header in request:
-            if string.lower(header).startswith("user-agent:"):
-                break
-        else:
-            request.append("User-Agent: %s" % self.user_agent)
-
-        return string.join(request, "\r\n") + "\r\n\r\n"
-
-
-    def handle_read(self):
-        # handle incoming data
-        data = self.recv(2048)
-
-        self.data = self.data + data
-
-        if len(data):
-            debug('got %d bytes from socket' % len(data))
-        else:
-            debug('server closed connection')
-
-        while self.data:
-            consumer = self.sent_requests[0][1]
-            if not self.response:
-                # do not have a full response header yet
-
-                # check if we've seen a full header
-                debug('getting header for %s' % self.sent_requests[0][0])
-
-                header = self.data.split("\r\n\r\n", 1)
-                if len(header) <= 1:
-                    return
-                header, self.data = header
-
-                self.response = _parse_response_http10(header)
-                self.content_remaining = self.response.content_length
-
-            if not self.data:
-                return
-
-            # we now know how many (more) content bytes we have, and how much
-            # is in the data buffer. there are two main possibilities:
-            # too much data, and some must be left behind containing the next
-            # response headers, or too little, or possibly just right
-
-            want = self.content_remaining
-            if want > 0:
-                got_data = self.data[:want]
-                self.data = self.data[want:]
-                
-                assert got_data
-
-                self.content_remaining -= len(got_data)
-
-                debug('pass back %d bytes of %s, %d remain'
-                      % (len(got_data),
-                         self.sent_requests[0][0],
-                         self.content_remaining))
-                consumer.feed(data)
-
-            if self.content_remaining == 0:
-                del self.sent_requests[0]
-
-                debug('content complete')
-                consumer.content_complete()
-                
-                # reset lots of things and try to get the next response header
-                if self.response.connection_reply == 'close':
-                    debug('server requested close')
-                    self.manager._channel_closed(self)
-                    self.close()
-                elif not self.manager.try_keepalive:
-                    debug('no keepalive for this socket')
-                    self.manager._channel_closed(self)
-                    self.close()
-                else:
-                    debug("ready for next header...")
-                    self.take_one()
-                self.response = None
-
-
-
-    def handle_close(self):
-        debug('async told us of close on %r' % self)
-        # if there are outstanding requests should probably reopen and 
-        # retransmit, but if we're not making any progress then give up
-        self.manager._channel_closed(self)
-        self.close()
-
-
-class DummyConsumer:
-    def __init__(self, url, pb):
-        self.url = url
-        self.outf = None
-        self._pb = pb
-
-    def feed(self, data):
-        # print "feed", repr(data)
-        # print "feed", repr(data[:20]), repr(data[-20:]), len(data)
-        if not self.outf:
-            base = self.url[self.url.rindex('/')+1:]
-            self.outf = file('/tmp/download/' + base, 'wb')
-        self.outf.write(data)
-
-    def error(self, err_info):
-        import traceback
-        error('error reported to consumer')
-        traceback.print_exception(err_info[0], err_info[1], err_info[2])
-        sys.exit(1)
-
-    def content_complete(self):
-        info('content complete from %s' % self.url)
-        self.outf.close()
-        self.outf = None
-        # using last_cnt is cheating
-        self._pb.update('downloading inventory',
-                        self._pb.last_cnt+1,
-                        self._pb.last_total)
-
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.DEBUG)
-
-    mgr = DownloadManager()
-
-    from bzrlib.branch import Branch
-    from bzrlib.progress import ProgressBar
-
-    pb = ProgressBar()
-    revs = Branch('/home/mbp/work/bzr').revision_history()
-    pb.update('downloading inventories', 0, len(revs))
-
-    for rev in revs:
-        url = 'http://www.bazaar-ng.org/bzr/bzr.dev/.bzr/inventory-store/' \
-              + rev + '.gz'
-        mgr.enqueue(url, DummyConsumer(url, pb))
-
-    mgr.run()
-    
-
-
-    
-#     for url in ['http://www.bazaar-ng.org/',
-#                 'http://www.bazaar-ng.org/tutorial.html',
-#                 'http://www.bazaar-ng.org/download.html',
-#                 'http://www.bazaar-ng.org/bzr/bzr.dev/.bzr/revision-store/mbp@hope-20050415013653-3b3c9c3d33fae0a6.gz',
-#                 ]:




More information about the bazaar-commits mailing list