Rev 3958: (jam) Add Transport._report_activity support to HTTP transports. in file:///home/pqm/archives/thelove/bzr/%2Btrunk/
Canonical.com Patch Queue Manager
pqm at pqm.ubuntu.com
Sat Jan 24 18:50:54 GMT 2009
At file:///home/pqm/archives/thelove/bzr/%2Btrunk/
------------------------------------------------------------
revno: 3958
revision-id: pqm at pqm.ubuntu.com-20090124185051-8oryvqq68n6repso
parent: pqm at pqm.ubuntu.com-20090124044940-7j90kl1qq22la0rx
parent: john at arbash-meinel.com-20090123212239-kbt36lwczrha6q67
committer: Canonical.com Patch Queue Manager <pqm at pqm.ubuntu.com>
branch nick: +trunk
timestamp: Sat 2009-01-24 18:50:51 +0000
message:
(jam) Add Transport._report_activity support to HTTP transports.
modified:
bzrlib/osutils.py osutils.py-20050309040759-eeaff12fbf77ac86
bzrlib/tests/test_osutils.py test_osutils.py-20051201224856-e48ee24c12182989
bzrlib/transport/http/__init__.py http_transport.py-20050711212304-506c5fd1059ace96
bzrlib/transport/http/_pycurl.py pycurlhttp.py-20060110060940-4e2a705911af77a6
bzrlib/transport/http/_urllib.py _urlgrabber.py-20060113083826-0bbf7d992fbf090c
bzrlib/transport/http/response.py _response.py-20060613154423-a2ci7hd4iw5c7fnt-1
------------------------------------------------------------
revno: 3956.2.2
revision-id: john at arbash-meinel.com-20090123212239-kbt36lwczrha6q67
parent: john at arbash-meinel.com-20090123212055-qsa1szqtrrxtwgdl
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 1.12-network-io
timestamp: Fri 2009-01-23 15:22:39 -0600
message:
Start using report_activity for HTTP (pycurl + urllib)
modified:
bzrlib/transport/http/__init__.py http_transport.py-20050711212304-506c5fd1059ace96
bzrlib/transport/http/_pycurl.py pycurlhttp.py-20060110060940-4e2a705911af77a6
bzrlib/transport/http/_urllib.py _urlgrabber.py-20060113083826-0bbf7d992fbf090c
bzrlib/transport/http/response.py _response.py-20060613154423-a2ci7hd4iw5c7fnt-1
------------------------------------------------------------
revno: 3956.2.1
revision-id: john at arbash-meinel.com-20090123212055-qsa1szqtrrxtwgdl
parent: pqm at pqm.ubuntu.com-20090123181416-tku4gdtorboy6d0y
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 1.12-network-io
timestamp: Fri 2009-01-23 15:20:55 -0600
message:
Add report_activity to osutils.pumpfile
modified:
bzrlib/osutils.py osutils.py-20050309040759-eeaff12fbf77ac86
bzrlib/tests/test_osutils.py test_osutils.py-20051201224856-e48ee24c12182989
=== modified file 'bzrlib/osutils.py'
--- a/bzrlib/osutils.py 2009-01-13 03:06:36 +0000
+++ b/bzrlib/osutils.py 2009-01-23 21:20:55 +0000
@@ -508,7 +508,8 @@
return False
-def pumpfile(from_file, to_file, read_length=-1, buff_size=32768):
+def pumpfile(from_file, to_file, read_length=-1, buff_size=32768,
+ report_activity=None, direction='read'):
"""Copy contents of one file to another.
The read_length can either be -1 to read to end-of-file (EOF) or
@@ -517,6 +518,10 @@
The buff_size represents the maximum size for each read operation
performed on from_file.
+ :param report_activity: Call this as bytes are read, see
+ Transport._report_activity
+ :param direction: Will be passed to report_activity
+
:return: The number of bytes copied.
"""
length = 0
@@ -530,6 +535,8 @@
if not block:
# EOF reached
break
+ if report_activity is not None:
+ report_activity(len(block), direction)
to_file.write(block)
actual_bytes_read = len(block)
@@ -542,6 +549,8 @@
if not block:
# EOF reached
break
+ if report_activity is not None:
+ report_activity(len(block), direction)
to_file.write(block)
length += len(block)
return length
=== modified file 'bzrlib/tests/test_osutils.py'
--- a/bzrlib/tests/test_osutils.py 2009-01-13 03:06:36 +0000
+++ b/bzrlib/tests/test_osutils.py 2009-01-23 21:20:55 +0000
@@ -486,6 +486,34 @@
message = "Data not equal. Expected %d bytes, received %d."
self.fail(message % (len(response_data), self.test_data_len))
+ def test_report_activity(self):
+ activity = []
+ def log_activity(length, direction):
+ activity.append((length, direction))
+ from_file = StringIO(self.test_data)
+ to_file = StringIO()
+ pumpfile(from_file, to_file, buff_size=500,
+ report_activity=log_activity, direction='read')
+ self.assertEqual([(500, 'read'), (500, 'read'), (500, 'read'),
+ (36, 'read')], activity)
+
+ from_file = StringIO(self.test_data)
+ to_file = StringIO()
+ del activity[:]
+ pumpfile(from_file, to_file, buff_size=500,
+ report_activity=log_activity, direction='write')
+ self.assertEqual([(500, 'write'), (500, 'write'), (500, 'write'),
+ (36, 'write')], activity)
+
+ # And with a limited amount of data
+ from_file = StringIO(self.test_data)
+ to_file = StringIO()
+ del activity[:]
+ pumpfile(from_file, to_file, buff_size=500, read_length=1028,
+ report_activity=log_activity, direction='read')
+ self.assertEqual([(500, 'read'), (500, 'read'), (28, 'read')], activity)
+
+
class TestPumpStringFile(TestCase):
=== modified file 'bzrlib/transport/http/__init__.py'
--- a/bzrlib/transport/http/__init__.py 2008-12-12 12:25:48 +0000
+++ b/bzrlib/transport/http/__init__.py 2009-01-23 21:22:39 +0000
@@ -123,13 +123,17 @@
:param relpath: The relative path to the file
"""
- code, response_file = self._get(relpath, None)
# FIXME: some callers want an iterable... One step forward, three steps
# backwards :-/ And not only an iterable, but an iterable that can be
# seeked backwards, so we will never be able to do that. One such
# known client is bzrlib.bundle.serializer.v4.get_bundle_reader. At the
# time of this writing it's even the only known client -- vila20071203
- return StringIO(response_file.read())
+ return StringIO(self.get_bytes(relpath))
+
+ def get_bytes(self, relpath):
+ """See Transport.get_bytes()."""
+ code, response_file = self._get(relpath, None)
+ return response_file.read()
def _get(self, relpath, ranges, tail_amount=0):
"""Get a file, or part of a file.
@@ -212,7 +216,6 @@
:param offsets: A list of (offset, size) tuples.
:param return: A list or generator of (offset, data) tuples
"""
-
# offsets may be a generator, we will iterate it several times, so
# build a list
offsets = list(offsets)
=== modified file 'bzrlib/transport/http/_pycurl.py'
--- a/bzrlib/transport/http/_pycurl.py 2008-12-12 14:33:13 +0000
+++ b/bzrlib/transport/http/_pycurl.py 2009-01-23 21:22:39 +0000
@@ -245,7 +245,8 @@
'Server return code %d'
% curl.getinfo(pycurl.HTTP_CODE))
msg = self._parse_headers(header)
- return code, response.handle_response(abspath, code, msg, data)
+ return code, response.handle_response(abspath, code, msg, data,
+ report_activity=self._report_activity)
def _parse_headers(self, status_and_headers):
"""Transform the headers provided by curl into an HTTPMessage"""
@@ -285,7 +286,9 @@
data.seek(0)
code = curl.getinfo(pycurl.HTTP_CODE)
msg = self._parse_headers(header)
- return code, response.handle_response(abspath, code, msg, data)
+ return code, response.handle_response(abspath, code, msg, data,
+ report_activity=self._report_activity)
+
def _raise_curl_http_error(self, curl, info=None):
code = curl.getinfo(pycurl.HTTP_CODE)
=== modified file 'bzrlib/transport/http/_urllib.py'
--- a/bzrlib/transport/http/_urllib.py 2008-12-12 14:33:13 +0000
+++ b/bzrlib/transport/http/_urllib.py 2009-01-23 21:22:39 +0000
@@ -126,7 +126,8 @@
raise errors.InvalidHttpRange(abspath, range_header,
'Server return code %d' % code)
- data = handle_response(abspath, code, response.info(), response)
+ data = handle_response(abspath, code, response.info(), response,
+ report_activity=self._report_activity)
return code, data
def _post(self, body_bytes):
@@ -136,7 +137,8 @@
response = self._perform(Request('POST', abspath, body_bytes,
accepted_errors=[200, 403]))
code = response.code
- data = handle_response(abspath, code, response.info(), response)
+ data = handle_response(abspath, code, response.info(), response,
+ report_activity=self._report_activity)
return code, data
def _head(self, relpath):
=== modified file 'bzrlib/transport/http/response.py'
--- a/bzrlib/transport/http/response.py 2008-07-15 13:11:53 +0000
+++ b/bzrlib/transport/http/response.py 2009-01-23 21:22:39 +0000
@@ -67,15 +67,18 @@
# maximum size of read requests -- used to avoid MemoryError issues in recv
_max_read_size = 512 * 1024
- def __init__(self, path, infile):
+ def __init__(self, path, infile, report_activity=None):
"""Constructor.
:param path: File url, for error reports.
:param infile: File-like socket set at body start.
+ :param report_activity: A Transport._report_activity function to call
+ as bytes are read.
"""
self._path = path
self._file = infile
self._boundary = None
+ self._report_activity = report_activity
# When using multi parts response, this will be set with the headers
# associated with the range currently read.
self._headers = None
@@ -228,7 +231,8 @@
limited = self._start + self._size - self._pos
if size >= 0:
limited = min(limited, size)
- osutils.pumpfile(self._file, buffer, limited, self._max_read_size)
+ osutils.pumpfile(self._file, buffer, limited, self._max_read_size,
+ report_activity=self._report_activity, direction='read')
data = buffer.getvalue()
# Update _pos respecting the data effectively read
@@ -277,7 +281,7 @@
return self._pos
-def handle_response(url, code, msg, data):
+def handle_response(url, code, msg, data, report_activity=None):
"""Interpret the code & headers and wrap the provided data in a RangeFile.
This is a factory method which returns an appropriate RangeFile based on
@@ -291,7 +295,7 @@
:return: A file-like object that can seek()+read() the
ranges indicated by the headers.
"""
- rfile = RangeFile(url, data)
+ rfile = RangeFile(url, data, report_activity=report_activity)
if code == 200:
# A whole file
size = msg.getheader('content-length', None)
More information about the bazaar-commits
mailing list