Rev 3907: _wire_bytes() now strips groups as necessary, as does _insert_record_stream in http://bzr.arbash-meinel.com/branches/bzr/brisbane/lazy_gc_stream

John Arbash Meinel john at arbash-meinel.com
Tue Mar 17 19:37:28 GMT 2009


At http://bzr.arbash-meinel.com/branches/bzr/brisbane/lazy_gc_stream

------------------------------------------------------------
revno: 3907
revision-id: john at arbash-meinel.com-20090317193722-lrb1bo81xp82y0nv
parent: john at arbash-meinel.com-20090317192705-8r4ny7purwsx3m0l
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: lazy_gc_stream
timestamp: Tue 2009-03-17 14:37:22 -0500
message:
  _wire_bytes() now strips groups as necessary, as does _insert_record_stream
-------------- next part --------------
=== modified file 'bzrlib/groupcompress.py'
--- a/bzrlib/groupcompress.py	2009-03-17 19:27:05 +0000
+++ b/bzrlib/groupcompress.py	2009-03-17 19:37:22 +0000
@@ -613,20 +613,7 @@
 
     def _wire_bytes(self):
         """Return a byte stream suitable for transmitting over the wire."""
-        # TODO: this might be a really good time to determine that we want to
-        #       rebuild a group based on how much content we are actually
-        #       transmitting.
-        #       Specifically, we could use max(end) to compare to
-        #       block._content_length to see if we want to just truncate the
-        #       last bytes. a simple:
-        #           last_needed_byte = max(end)
-        #           bytes = zlib.decompress(last_needed_byte+XXX)
-        #           c_bytes = zlib.compress(bytes[:last_needed_byte])
-        #       Going further, we could compare sum(end-start) to see how many
-        #       bytes out of the partial content we will actually be using, and
-        #       use a heuristic to decide that we need to generate a new group.
-        #       (that could be if *any* bytes are unused, or just if more than
-        #       XX percent is unused)
+        self._check_rebuild_block()
         # The outer block starts with:
         #   'groupcompress-block\n'
         #   <length of compressed key info>\n
@@ -660,7 +647,6 @@
         z_header_bytes = zlib.compress(header_bytes)
         del header_bytes
         z_header_bytes_len = len(z_header_bytes)
-        assert self._block._z_content is not None
         block_bytes = self._block.to_bytes()
         lines.append('%d\n%d\n%d\n' % (z_header_bytes_len, header_bytes_len,
                                        len(block_bytes)))
@@ -1426,6 +1412,7 @@
                 if record.storage_kind == 'groupcompress-block':
                     # Insert the raw block into the target repo
                     insert_manager = record._manager
+                    record._manager._check_rebuild_block()
                     bytes = record._manager._block.to_bytes()
                     _, start, length = self._access.add_raw_records(
                         [(None, len(bytes))], bytes)[0]

=== modified file 'bzrlib/repofmt/groupcompress_repo.py'
--- a/bzrlib/repofmt/groupcompress_repo.py	2009-03-17 18:37:51 +0000
+++ b/bzrlib/repofmt/groupcompress_repo.py	2009-03-17 19:37:22 +0000
@@ -381,9 +381,7 @@
         child_pb = ui.ui_factory.nested_progress_bar()
         try:
             stream = vf_to_stream(source_vf, keys, message, child_pb)
-            for _ in target_vf._insert_record_stream(stream,
-                                                     reuse_blocks=False):
-                pass
+            target_vf.insert_record_stream(stream)
         finally:
             child_pb.finished()
 
@@ -414,9 +412,7 @@
         try:
             for stream in self._get_chk_streams(source_vf, total_keys,
                                                 pb=child_pb):
-                for _ in target_vf._insert_record_stream(stream,
-                                                         reuse_blocks=False):
-                    pass
+                target_vf.insert_record_stream(stream)
         finally:
             child_pb.finished()
 

=== modified file 'bzrlib/tests/test_groupcompress.py'
--- a/bzrlib/tests/test_groupcompress.py	2009-03-17 19:27:05 +0000
+++ b/bzrlib/tests/test_groupcompress.py	2009-03-17 19:37:22 +0000
@@ -636,13 +636,19 @@
         manager = groupcompress._LazyGroupContentManager(block)
         wire_bytes = manager._wire_bytes()
         block_length = len(block.to_bytes())
-        self.assertStartsWith(wire_bytes,
-                              'groupcompress-block\n'
-                              '8\n' # len(compress(''))
-                              '0\n' # len('')
-                              '%d\n'
-                              % (block_length,)
-                              )
+        # We should have triggered a strip, since we aren't using any content
+        stripped_block = manager._block.to_bytes()
+        self.assertTrue(block_length > len(stripped_block))
+        empty_z_header = zlib.compress('')
+        self.assertEqual('groupcompress-block\n'
+                         '8\n' # len(compress(''))
+                         '0\n' # len('')
+                         '%d\n'# compressed block len
+                         '%s'  # zheader
+                         '%s'  # block
+                         % (len(stripped_block), empty_z_header,
+                            stripped_block),
+                         wire_bytes)
 
     def test__wire_bytes(self):
         entries, block = self.make_block(self._texts)



More information about the bazaar-commits mailing list