Rev 2677: Remove the special code for _extract_lines_from_gzip in http://bzr.arbash-meinel.com/branches/bzr/0.19-dev/pyrex_knit_extract
John Arbash Meinel
john at arbash-meinel.com
Fri Aug 3 00:33:27 BST 2007
At http://bzr.arbash-meinel.com/branches/bzr/0.19-dev/pyrex_knit_extract
------------------------------------------------------------
revno: 2677
revision-id: john at arbash-meinel.com-20070802233253-jksnt66mv2ti5mhi
parent: john at arbash-meinel.com-20070802232606-z9rvcyiayw4grg0u
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: pyrex_knit_extract
timestamp: Thu 2007-08-02 18:32:53 -0500
message:
Remove the special code for _extract_lines_from_gzip
since we don't use it directly anymore.
modified:
bzrlib/knit.py knit.py-20051212171256-f056ac8f0fbe1bd9
bzrlib/tests/test__knit_helpers.py test__knit_helpers.p-20070724214127-vkhtjicq8lrlotz3-1
-------------- next part --------------
=== modified file 'bzrlib/knit.py'
--- a/bzrlib/knit.py 2007-07-27 20:11:47 +0000
+++ b/bzrlib/knit.py 2007-08-02 23:32:53 +0000
@@ -1415,28 +1415,6 @@
return [parent for parent in self._cache[version_id][4]
if parent in self._cache]
- def get_build_chain(self, version_id):
- """Get the chain that we need to get a fulltext for this version.
-
- :return: [(version_id, start, size)] needed to extract this text.
- """
- cursor = version_id
- chain = []
- while cursor is not None:
- v_id, options, pos, size, parents, idx = self._cache[cursor]
- chain.append((v_id, pos, size))
- if 'fulltext' in options:
- cursor = None
- elif 'line-delta' not in options:
- raise errors.KnitIndexUnknownMethod(self._full_path(), options)
- else:
- # Line-delta is based on the first present parent
- # Do we have to handle missing the primary parent?
- cursor = parents[0]
- # Put the fulltext first
- chain.reverse()
- return chain
-
def get_parents_with_ghosts(self, version_id):
"""Return parents of specified version with ghosts."""
return self._cache[version_id][4]
@@ -1844,8 +1822,14 @@
return df, rec
def _parse_record(self, version_id, data):
+ # profiling notes:
+ # 4168 calls in 2880 217 internal
+ # 4168 calls to _parse_record_header in 2121
+ # 4168 calls to readlines in 330
+ df = GzipFile(mode='rb', fileobj=StringIO(data))
+
try:
- record_contents = _extract_lines_from_gzip(data)
+ record_contents = df.readlines()
except Exception, e:
raise KnitCorrupt(self._filename,
"While reading {%s} got %s(%s)"
@@ -1864,6 +1848,7 @@
raise KnitCorrupt(self._filename,
'unexpected version end line %r, wanted %r'
% (last_line, version_id))
+ df.close()
return record_contents, rec[3]
def read_records_iter_raw(self, records):
@@ -2326,14 +2311,12 @@
try:
from bzrlib._knit_helpers_c import (
_load_data_c as _load_data,
- _extract_lines_from_gzip_c as _extract_lines_from_gzip,
_extract_knit_fulltext_from_gzip_c as _extract_knit_fulltext_from_gzip,
_extract_knit_linedelta_from_gzip_c as _extract_knit_linedelta_from_gzip,
)
except ImportError:
from bzrlib._knit_helpers_py import (
_load_data_py as _load_data,
- _extract_lines_from_gzip_py as _extract_lines_from_gzip,
_extract_knit_fulltext_from_gzip_py as _extract_knit_fulltext_from_gzip,
_extract_knit_linedelta_from_gzip_py as _extract_knit_linedelta_from_gzip,
)
=== modified file 'bzrlib/tests/test__knit_helpers.py'
--- a/bzrlib/tests/test__knit_helpers.py 2007-08-02 21:55:05 +0000
+++ b/bzrlib/tests/test__knit_helpers.py 2007-08-02 23:32:53 +0000
@@ -43,58 +43,6 @@
return sio.getvalue()
-class TestExtractLines(tests.TestCase):
- """We should be able to extract the lines from a gzip stream."""
-
- def get_extract_lines_from_gzip(self):
- """Get an implementation of extract lines.
-
- Child classes can override this to provide custom implementations.
- """
- return _knit_helpers_py._extract_lines_from_gzip_py
-
- def test_get_lines(self):
- gz_value = gzip_compress('a\nbunch\nof\nlines\n')
- extract_lines = self.get_extract_lines_from_gzip()
- value = extract_lines(gz_value)
- self.assertEqual(['a\n', 'bunch\n', 'of\n', 'lines\n'],
- value)
-
- def test_with_carriage_return(self):
- """make sure it works correctly if '\\r' is in the stream"""
- gz_value = gzip_compress('some\rtext\rwith\ncarriage return\n')
- extract_lines = self.get_extract_lines_from_gzip()
- value = extract_lines(gz_value)
- self.assertEqual(['some\rtext\rwith\n', 'carriage return\n'],
- value)
-
- def test_large_buffer(self):
- """A really long buffer, to watch out for mem errors."""
- # A buffer of 256 characters
- chars = [chr(x) for x in xrange(256) if x != 10]
- chars.append('\n')
- chars = ''.join(chars)
- out = []
- # Compress 10MB of data
- for i in xrange(10240):
- out.append(chars)
- gz_value = gzip_compress(''.join(out))
- extract_lines = self.get_extract_lines_from_gzip()
- value = extract_lines(gz_value)
- # Don't use assertEqual because we don't want to spew 10MB of data to
- # the screen
- if value != out:
- self.fail('Decompressing a 10MB buffer failed.')
-
-
-class TestExtractLinesCompiled(TestExtractLines):
-
- _test_needs_features = [CompiledKnitFeature]
-
- def get_extract_lines_from_gzip(self):
- return _knit_helpers_c._extract_lines_from_gzip_c
-
-
class TestExtractKnitFulltextFromGzip(tests.TestCase):
"""Test implementations of _extract_knit_fulltext_from_gzip.
@@ -281,12 +229,24 @@
from bzrlib import _knit_helpers_py
self.assertIs(_knit_helpers_py._load_data_py, knit._load_data)
- def test_extract_lines_from_gzip(self):
- if CompiledKnitFeature.available():
- from bzrlib import _knit_helpers_c
- self.assertIs(_knit_helpers_c._extract_lines_from_gzip_c,
- knit._extract_lines_from_gzip)
- else:
- from bzrlib import _knit_helpers_py
- self.assertIs(_knit_helpers_py._extract_lines_from_gzip_py,
- knit._extract_lines_from_gzip)
+ def test_extract_knit_fulltext_from_gzip(self):
+ if CompiledKnitFeature.available():
+ from bzrlib import _knit_helpers_c
+ self.assertIs(_knit_helpers_c._extract_knit_fulltext_from_gzip_c,
+ knit._extract_knit_fulltext_from_gzip)
+ else:
+ from bzrlib import _knit_helpers_py
+ self.assertIs(_knit_helpers_py._extract_knit_fulltext_from_gzip_py,
+ knit._extract_knit_fulltext_from_gzip)
+
+ def test_extract_knit_linedelta_from_gzip(self):
+ if CompiledKnitFeature.available():
+ from bzrlib import _knit_helpers_c
+ self.assertIs(
+ _knit_helpers_c._extract_knit_linedelta_from_gzip_c,
+ knit._extract_knit_linedelta_from_gzip)
+ else:
+ from bzrlib import _knit_helpers_py
+ self.assertIs(
+ _knit_helpers_py._extract_knit_linedelta_from_gzip_py,
+ knit._extract_knit_linedelta_from_gzip)
More information about the bazaar-commits
mailing list