Rev 3202: Convert the get_parent_map code to gzip the result, allowing more revisions on the wire. in http://people.ubuntu.com/~robertc/baz2.0/search-results

[Robert Collins]

At http://people.ubuntu.com/~robertc/baz2.0/search-results

------------------------------------------------------------
revno: 3202
revision-id:robertc at robertcollins.net-20080203155204-lw3zej7zdfzwdgcs
parent: robertc at robertcollins.net-20080203150131-v1od7axpjo43ptho
committer: Robert Collins <robertc at robertcollins.net>
branch nick: Remote_missing_uses_search
timestamp: Mon 2008-02-04 02:52:04 +1100
message:
  Convert the get_parent_map code to gzip the result, allowing more revisions on the wire.
modified:
  bzrlib/remote.py               remote.py-20060720103555-yeeg2x51vn0rbtdp-1
  bzrlib/smart/repository.py     repository.py-20061128022038-vr5wy5bubyb8xttk-1
  bzrlib/tests/test_remote.py    test_remote.py-20060720103555-yeeg2x51vn0rbtdp-2
  bzrlib/tests/test_smart.py     test_smart.py-20061122024551-ol0l0o0oofsu9b3t-2
=== modified file 'bzrlib/remote.py'
--- a/bzrlib/remote.py	2008-02-03 15:01:31 +0000
+++ b/bzrlib/remote.py	2008-02-03 15:52:04 +0000
@@ -42,6 +42,7 @@
     )
 from bzrlib.revision import NULL_REVISION
 from bzrlib.trace import mutter, note
+from bzrlib.tuned_gzip import GzipFile
 
 # Note: RemoteBzrDirFormat is in bzrdir.py
 
@@ -838,7 +839,8 @@
             reponse[1].cancel_read_body()
             raise errors.UnexpectedSmartServerResponse(response[0])
         if response[0][0] == 'ok':
-            coded = response[1].read_body_bytes()
+            coded = GzipFile(mode='rb',
+                fileobj=StringIO(response[1].read_body_bytes())).read()
             if coded == '':
                 # no revisions found
                 return {}

=== modified file 'bzrlib/smart/repository.py'
--- a/bzrlib/smart/repository.py	2008-01-18 05:27:43 +0000
+++ b/bzrlib/smart/repository.py	2008-02-03 15:52:04 +0000
@@ -31,6 +31,7 @@
     SuccessfulSmartServerResponse,
     )
 from bzrlib import revision as _mod_revision
+from bzrlib.tuned_gzip import GzipFile, bytes_to_gzip
 
 
 class SmartServerRepositoryRequest(SmartServerRequest):
@@ -107,8 +108,8 @@
         """Process the current search state and perform the parent lookup.
 
         :return: A smart server response where the body contains an utf8
-            encoded flattened list of the parents of the revisions, (the same
-            format as Repository.get_revision_graph).
+            encoded flattened list of the parents of the revisions (the same
+            format as Repository.get_revision_graph) which has been gzipped.
         """
         repository = self._repository
         repository.lock_read()
@@ -152,19 +153,23 @@
                     # Approximate the serialized cost of this revision_id.
                     size_so_far += 2 + len(revision_id) + sum(map(len, parents))
             # get all the directly asked for parents, and then flesh out to
-            # 64K or so. We do one level of depth at a time to stay in sync
-            # with the client.
-            if first_loop_done and size_so_far > 65000:
+            # 64K (compressed) or so. We do one level of depth at a time to
+            # stay in sync with the client. The 185000 magic number is
+            # estimated compression ratio taken from bzr.dev itself.
+            if first_loop_done and size_so_far > 185000:
                 next_revs = set()
                 break
             # don't query things we've already queried
             next_revs.difference_update(queried_revs)
             first_loop_done = True
 
-        for revision, parents in result.items():
+        # sorting trivially puts lexographically similar revision ids together.
+        # Compression FTW.
+        for revision, parents in sorted(result.items()):
             lines.append(' '.join((revision, ) + tuple(parents)))
 
-        return SuccessfulSmartServerResponse(('ok', ), '\n'.join(lines))
+        return SuccessfulSmartServerResponse(
+            ('ok', ), bytes_to_gzip('\n'.join(lines)))
 
 
 class SmartServerRepositoryGetRevisionGraph(SmartServerRepositoryRequest):

=== modified file 'bzrlib/tests/test_remote.py'
--- a/bzrlib/tests/test_remote.py	2008-01-31 03:42:10 +0000
+++ b/bzrlib/tests/test_remote.py	2008-02-03 15:52:04 +0000
@@ -46,6 +46,7 @@
 from bzrlib.smart.client import _SmartClient
 from bzrlib.transport.memory import MemoryTransport
 from bzrlib.transport.remote import RemoteTransport
+from bzrlib.tuned_gzip import bytes_to_gzip
 
 
 class BasicRemoteObjectTests(tests.TestCaseWithTransport):
@@ -608,7 +609,7 @@
         r1 = u'\u0e33'.encode('utf8')
         r2 = u'\u0dab'.encode('utf8')
         lines = [' '.join([r2, r1]), r1]
-        encoded_body = '\n'.join(lines)
+        encoded_body = bytes_to_gzip('\n'.join(lines))
         responses = [(('ok', ), encoded_body), (('ok', ), encoded_body)]
 
         transport_path = 'quack'

=== modified file 'bzrlib/tests/test_smart.py'
--- a/bzrlib/tests/test_smart.py	2008-01-17 07:49:09 +0000
+++ b/bzrlib/tests/test_smart.py	2008-02-03 15:52:04 +0000
@@ -37,6 +37,7 @@
 import bzrlib.smart.bzrdir
 import bzrlib.smart.branch
 import bzrlib.smart.repository
+from bzrlib.tuned_gzip import bytes_to_gzip
 from bzrlib.util import bencode
 
 
@@ -529,6 +530,22 @@
             request.execute, backing.local_abspath('subdir'))
 
 
+class TestSmartServerRepositoryGetParentMap(tests.TestCaseWithTransport):
+
+    def test_trivial_gzipped(self):
+        # This tests that the wire encoding is actually gzipped
+        backing = self.get_transport()
+        request = smart.repository.SmartServerRepositoryGetParentMap(backing)
+        tree = self.make_branch_and_memory_tree('.')
+
+        self.assertEqual(None,
+            request.execute(backing.local_abspath(''), 'missing-id'))
+        # Note that it returns a body (of '' gzipped).
+        self.assertEqual(
+            SuccessfulSmartServerResponse(('ok', ), bytes_to_gzip('')),
+            request.do_body('\n\n0\n'))
+
+
 class TestSmartServerRepositoryGetRevisionGraph(tests.TestCaseWithTransport):
 
     def test_none_argument(self):