Rev 2663: * ``bzrlib.pack.make_readv_reader`` allows readv based access to pack in http://people.ubuntu.com/~robertc/baz2.0/pack
Robert Collins
robertc at robertcollins.net
Thu Aug 2 04:18:06 BST 2007
At http://people.ubuntu.com/~robertc/baz2.0/pack
------------------------------------------------------------
revno: 2663
revision-id: robertc at robertcollins.net-20070802031746-mpnoaxym829719w6
parent: robertc at robertcollins.net-20070802021817-n8a86kevyvk2f9jo
committer: Robert Collins <robertc at robertcollins.net>
branch nick: pack
timestamp: Thu 2007-08-02 13:17:46 +1000
message:
* ``bzrlib.pack.make_readv_reader`` allows readv based access to pack
files that are stored on a transport. (Robert Collins)
modified:
NEWS NEWS-20050323055033-4e00b5db738777ff
bzrlib/pack.py container.py-20070607160755-tr8zc26q18rn0jnb-1
bzrlib/tests/test_pack.py test_container.py-20070607160755-tr8zc26q18rn0jnb-2
bzrlib/transport/local.py local_transport.py-20050711165921-9b1f142bfe480c24
=== modified file 'NEWS'
--- a/NEWS 2007-08-02 02:18:17 +0000
+++ b/NEWS 2007-08-02 03:17:46 +0000
@@ -182,6 +182,9 @@
during pack creation, without needing a separate pass across the output
pack to gather such details. (Robert Collins)
+ * ``bzrlib.pack.make_readv_reader`` allows readv based access to pack
+ files that are stored on a transport. (Robert Collins)
+
TESTING:
* Remove selftest ``--clean-output``, ``--numbered-dirs`` and
=== modified file 'bzrlib/pack.py'
--- a/bzrlib/pack.py 2007-08-02 02:18:17 +0000
+++ b/bzrlib/pack.py 2007-08-02 03:17:46 +0000
@@ -19,6 +19,7 @@
"Containers" and "records" are described in doc/developers/container-format.txt.
"""
+from cStringIO import StringIO
import re
from bzrlib import errors
@@ -112,6 +113,52 @@
return current_offset, self.current_offset - current_offset
+class ReadVFile(object):
+ """Adapt a readv result iterator to a file like protocol."""
+
+ def __init__(self, readv_result):
+ self.readv_result = readv_result
+ # the most recent readv result block
+ self._string = None
+
+ def _next(self):
+ if (self._string is None or
+ self._string.tell() == self._string_length):
+ length, data = self.readv_result.next()
+ self._string_length = len(data)
+ self._string = StringIO(data)
+
+ def read(self, length):
+ self._next()
+ result = self._string.read(length)
+ if len(result) < length:
+ raise errors.BzrError('request for too much data from a readv hunk.')
+ return result
+
+ def readline(self):
+ """Note that readline will not cross readv segments."""
+ self._next()
+ result = self._string.readline()
+ if self._string.tell() == self._string_length and result[-1] != '\n':
+ raise errors.BzrError('short readline in the readvfile hunk.')
+ return result
+
+
+def make_readv_reader(transport, filename, requested_records):
+ """Create a ContainerReader that will read selected records only.
+
+ :param transport: The transport the pack file is located on.
+ :param filename: The filename of the pack file.
+ :param requested_records: The record offset, length tuples as returned
+ by add_bytes_record for the desired records.
+ """
+ readv_blocks = [(0, len(FORMAT_ONE)+1)]
+ readv_blocks.extend(requested_records)
+ result = ContainerReader(ReadVFile(
+ transport.readv(filename, readv_blocks)))
+ return result
+
+
class BaseReader(object):
def __init__(self, source_file):
=== modified file 'bzrlib/tests/test_pack.py'
--- a/bzrlib/tests/test_pack.py 2007-08-02 02:18:17 +0000
+++ b/bzrlib/tests/test_pack.py 2007-08-02 03:17:46 +0000
@@ -391,3 +391,63 @@
self.assertEqual('', get_bytes(99))
+class TestReadvReader(tests.TestCaseWithTransport):
+
+ def test_read_skipping_records(self):
+ pack_data = StringIO()
+ writer = pack.ContainerWriter(pack_data.write)
+ writer.begin()
+ memos = []
+ memos.append(writer.add_bytes_record('abc', names=[]))
+ memos.append(writer.add_bytes_record('def', names=['name1']))
+ memos.append(writer.add_bytes_record('ghi', names=['name2']))
+ memos.append(writer.add_bytes_record('jkl', names=[]))
+ writer.end()
+ transport = self.get_transport()
+ pack_data.seek(0)
+ transport.put_file('mypack', pack_data)
+ requested_records = [memos[0], memos[2]]
+ reader = pack.make_readv_reader(transport, 'mypack', requested_records)
+ result = []
+ for names, reader_func in reader.iter_records():
+ result.append((names, reader_func(None)))
+ self.assertEqual([([], 'abc'), (['name2'], 'ghi')], result)
+
+
+class TestReadvFile(tests.TestCaseWithTransport):
+
+ def test_read_bytes(self):
+ # this tests byte reading - solo, all in a hunk at once
+ transport = self.get_transport()
+ transport.put_bytes('sample', '0123456789')
+ f = pack.ReadVFile(transport.readv('sample', [(0,1), (1,2), (4,1), (6,2)]))
+ results = []
+ results.append(f.read(1))
+ results.append(f.read(2))
+ results.append(f.read(1))
+ results.append(f.read(1))
+ results.append(f.read(1))
+ self.assertEqual(['0', '12', '4', '6', '7'], results)
+
+ def test_readline(self):
+ # this tests readline as the ContainerReader needs it -
+ # just within a record, never across.
+ transport = self.get_transport()
+ transport.put_bytes('sample', '0\n2\n4\n')
+ f = pack.ReadVFile(transport.readv('sample', [(0,2), (2,4)]))
+ results = []
+ results.append(f.readline())
+ results.append(f.readline())
+ results.append(f.readline())
+ self.assertEqual(['0\n', '2\n', '4\n'], results)
+
+ def test_readline_and_read(self):
+ # this tests read, then readline, then read again
+ transport = self.get_transport()
+ transport.put_bytes('sample', '0\n2\n4\n')
+ f = pack.ReadVFile(transport.readv('sample', [(0,6)]))
+ results = []
+ results.append(f.read(1))
+ results.append(f.readline())
+ results.append(f.read(4))
+ self.assertEqual(['0', '\n', '2\n4\n'], results)
=== modified file 'bzrlib/transport/local.py'
--- a/bzrlib/transport/local.py 2007-07-20 03:20:20 +0000
+++ b/bzrlib/transport/local.py 2007-08-02 03:17:46 +0000
@@ -127,7 +127,7 @@
abspath = u'.'
return urlutils.file_relpath(
- urlutils.strip_trailing_slash(self.base),
+ urlutils.strip_trailing_slash(self.base),
urlutils.strip_trailing_slash(abspath))
def has(self, relpath):
More information about the bazaar-commits
mailing list