Rev 6314: (jelmer) Add a ``Repository.iter_revisions`` HPSS call. (Jelmer Vernooij) in file:///srv/pqm.bazaar-vcs.org/archives/thelove/bzr/%2Btrunk/

Patch Queue Manager pqm at pqm.ubuntu.com
Mon Nov 28 13:54:34 UTC 2011


At file:///srv/pqm.bazaar-vcs.org/archives/thelove/bzr/%2Btrunk/

------------------------------------------------------------
revno: 6314 [merge]
revision-id: pqm at pqm.ubuntu.com-20111128135433-xoddhlxx9qgi4u5k
parent: pqm at pqm.ubuntu.com-20111128121442-e21724bjf6i6d5xo
parent: jelmer at samba.org-20111128130515-s29pwv14nl5w4mbl
committer: Patch Queue Manager <pqm at pqm.ubuntu.com>
branch nick: +trunk
timestamp: Mon 2011-11-28 13:54:33 +0000
message:
  (jelmer) Add a ``Repository.iter_revisions`` HPSS call. (Jelmer Vernooij)
modified:
  bzrlib/remote.py               remote.py-20060720103555-yeeg2x51vn0rbtdp-1
  bzrlib/smart/repository.py     repository.py-20061128022038-vr5wy5bubyb8xttk-1
  bzrlib/smart/request.py        request.py-20061108095550-gunadhxmzkdjfeek-1
  bzrlib/tests/blackbox/test_log.py test_log.py-20060112090212-78f6ea560c868e24
  bzrlib/tests/blackbox/test_sign_my_commits.py test_sign_my_commits.py-20060215152957-270238a1ffacc841
  bzrlib/tests/test_remote.py    test_remote.py-20060720103555-yeeg2x51vn0rbtdp-2
  bzrlib/tests/test_smart.py     test_smart.py-20061122024551-ol0l0o0oofsu9b3t-2
  doc/en/release-notes/bzr-2.5.txt bzr2.5.txt-20110708125756-587p0hpw7oke4h05-1
=== modified file 'bzrlib/remote.py'
--- a/bzrlib/remote.py	2011-11-28 11:49:43 +0000
+++ b/bzrlib/remote.py	2011-11-28 13:05:15 +0000
@@ -15,6 +15,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
 import bz2
+import zlib
 
 from bzrlib import (
     bencode,
@@ -51,6 +52,7 @@
 from bzrlib.revision import NULL_REVISION
 from bzrlib.revisiontree import InventoryRevisionTree
 from bzrlib.repository import RepositoryWriteLockResult, _LazyListJoin
+from bzrlib.serializer import format_registry as serializer_format_registry
 from bzrlib.trace import mutter, note, warning, log_exception_quietly
 
 
@@ -2229,10 +2231,69 @@
         self._ensure_real()
         return self._real_repository.texts
 
+    def _iter_revisions_rpc(self, revision_ids):
+        body = "\n".join(revision_ids)
+        path = self.bzrdir._path_for_remote_call(self._client)
+        response_tuple, response_handler = (
+            self._call_with_body_bytes_expecting_body(
+            "Repository.iter_revisions", (path, ), body))
+        if response_tuple[0] != "ok":
+            raise errors.UnexpectedSmartServerResponse(response_tuple)
+        serializer_format = response_tuple[1]
+        serializer = serializer_format_registry.get(serializer_format)
+        byte_stream = response_handler.read_streamed_body()
+        decompressor = zlib.decompressobj()
+        chunks = []
+        for bytes in byte_stream:
+            chunks.append(decompressor.decompress(bytes))
+            if decompressor.unused_data != "":
+                chunks.append(decompressor.flush())
+                yield serializer.read_revision_from_string("".join(chunks))
+                unused = decompressor.unused_data
+                decompressor = zlib.decompressobj()
+                chunks = [decompressor.decompress(unused)]
+        chunks.append(decompressor.flush())
+        text = "".join(chunks)
+        if text != "":
+            yield serializer.read_revision_from_string("".join(chunks))
+
     @needs_read_lock
     def get_revisions(self, revision_ids):
-        self._ensure_real()
-        return self._real_repository.get_revisions(revision_ids)
+        if revision_ids is None:
+            revision_ids = self.all_revision_ids()
+        else:
+            for rev_id in revision_ids:
+                if not rev_id or not isinstance(rev_id, basestring):
+                    raise errors.InvalidRevisionId(
+                        revision_id=rev_id, branch=self)
+        try:
+            missing = set(revision_ids)
+            revs = {}
+            for rev in self._iter_revisions_rpc(revision_ids):
+                missing.remove(rev.revision_id)
+                revs[rev.revision_id] = rev
+        except errors.UnknownSmartMethod:
+            self._ensure_real()
+            return self._real_repository.get_revisions(revision_ids)
+        for fallback in self._fallback_repositories:
+            if not missing:
+                break
+            for revid in list(missing):
+                # XXX JRV 2011-11-20: It would be nice if there was a
+                # public method on Repository that could be used to query
+                # for revision objects *without* failing completely if one
+                # was missing. There is VersionedFileRepository._iter_revisions,
+                # but unfortunately that's private and not provided by
+                # all repository implementations.
+                try:
+                    revs[revid] = fallback.get_revision(revid)
+                except errors.NoSuchRevision:
+                    pass
+                else:
+                    missing.remove(revid)
+        if missing:
+            raise errors.NoSuchRevision(self, list(missing)[0])
+        return [revs[revid] for revid in revision_ids]
 
     def supports_rich_root(self):
         return self._format.rich_root_data

=== modified file 'bzrlib/smart/repository.py'
--- a/bzrlib/smart/repository.py	2011-11-28 03:38:43 +0000
+++ b/bzrlib/smart/repository.py	2011-11-28 11:34:18 +0000
@@ -22,6 +22,7 @@
 import sys
 import tempfile
 import threading
+import zlib
 
 from bzrlib import (
     bencode,
@@ -1101,3 +1102,39 @@
         finally:
             self._repository.unlock()
         return SuccessfulSmartServerResponse(("ok", ), )
+
+
+class SmartServerRepositoryIterRevisions(SmartServerRepositoryRequest):
+    """Stream a list of revisions.
+
+    The client sends a list of newline-separated revision ids in the
+    body of the request and the server replies with the serializer format,
+    and a stream of bzip2-compressed revision texts (using the specified
+    serializer format).
+
+    Any revisions the server does not have are omitted from the stream.
+
+    New in 2.5.
+    """
+
+    def do_repository_request(self, repository):
+        self._repository = repository
+        # Signal there is a body
+        return None
+
+    def do_body(self, body_bytes):
+        revision_ids = body_bytes.split("\n")
+        return SuccessfulSmartServerResponse(
+            ('ok', self._repository.get_serializer_format()),
+            body_stream=self.body_stream(self._repository, revision_ids))
+
+    def body_stream(self, repository, revision_ids):
+        self._repository.lock_read()
+        try:
+            for record in repository.revisions.get_record_stream(
+                [(revid,) for revid in revision_ids], 'unordered', True):
+                if record.storage_kind == 'absent':
+                    continue
+                yield zlib.compress(record.get_bytes_as('fulltext'))
+        finally:
+            self._repository.unlock()

=== modified file 'bzrlib/smart/request.py'
--- a/bzrlib/smart/request.py	2011-11-28 03:38:43 +0000
+++ b/bzrlib/smart/request.py	2011-11-28 11:34:18 +0000
@@ -676,6 +676,9 @@
     'Repository.get_stream_1.19', 'bzrlib.smart.repository',
     'SmartServerRepositoryGetStream_1_19')
 request_handlers.register_lazy(
+    'Repository.iter_revisions', 'bzrlib.smart.repository',
+    'SmartServerRepositoryIterRevisions')
+request_handlers.register_lazy(
     'Repository.pack', 'bzrlib.smart.repository',
     'SmartServerRepositoryPack')
 request_handlers.register_lazy(

=== modified file 'bzrlib/tests/blackbox/test_log.py'
--- a/bzrlib/tests/blackbox/test_log.py	2011-11-23 11:22:56 +0000
+++ b/bzrlib/tests/blackbox/test_log.py	2011-11-23 12:00:17 +0000
@@ -1069,7 +1069,7 @@
         # being too low. If rpc_count increases, more network roundtrips have
         # become necessary for this use case. Please do not adjust this number
         # upwards without agreement from bzr's network support maintainers.
-        self.assertLength(15, self.hpss_calls)
+        self.assertLength(10, self.hpss_calls)
 
     def test_verbose_log(self):
         self.setup_smart_server_with_call_log()
@@ -1084,4 +1084,4 @@
         # being too low. If rpc_count increases, more network roundtrips have
         # become necessary for this use case. Please do not adjust this number
         # upwards without agreement from bzr's network support maintainers.
-        self.assertLength(20, self.hpss_calls)
+        self.assertLength(19, self.hpss_calls)

=== modified file 'bzrlib/tests/blackbox/test_sign_my_commits.py'
--- a/bzrlib/tests/blackbox/test_sign_my_commits.py	2011-11-28 11:17:05 +0000
+++ b/bzrlib/tests/blackbox/test_sign_my_commits.py	2011-11-28 13:05:15 +0000
@@ -164,7 +164,7 @@
         # being too low. If rpc_count increases, more network roundtrips have
         # become necessary for this use case. Please do not adjust this number
         # upwards without agreement from bzr's network support maintainers.
-        self.assertLength(53, self.hpss_calls)
+        self.assertLength(54, self.hpss_calls)
 
     def test_verify_commits(self):
         self.setup_smart_server_with_call_log()
@@ -186,6 +186,6 @@
         # The number of readv requests seems to vary depending on the generated
         # repository and how well it compresses, so allow for a bit of
         # variation:
-        if len(self.hpss_calls) not in (21, 22):
-            self.fail("Incorrect length: wanted 21 or 22, got %d for %r" % (
+        if len(self.hpss_calls) not in (20, 21):
+            self.fail("Incorrect length: wanted 20 or 21, got %d for %r" % (
                 len(self.hpss_calls), self.hpss_calls))

=== modified file 'bzrlib/tests/test_remote.py'
--- a/bzrlib/tests/test_remote.py	2011-11-28 11:49:43 +0000
+++ b/bzrlib/tests/test_remote.py	2011-11-28 13:05:15 +0000
@@ -25,6 +25,7 @@
 
 import bz2
 from cStringIO import StringIO
+import zlib
 
 from bzrlib import (
     branch,
@@ -48,6 +49,7 @@
     BzrDirFormat,
     RemoteBzrProber,
     )
+from bzrlib.chk_serializer import chk_bencode_serializer
 from bzrlib.remote import (
     RemoteBranch,
     RemoteBranchFormat,
@@ -57,7 +59,10 @@
     RemoteRepositoryFormat,
     )
 from bzrlib.repofmt import groupcompress_repo, knitpack_repo
-from bzrlib.revision import NULL_REVISION
+from bzrlib.revision import (
+    NULL_REVISION,
+    Revision,
+    )
 from bzrlib.smart import medium, request
 from bzrlib.smart.client import _SmartClient
 from bzrlib.smart.repository import (
@@ -2620,6 +2625,43 @@
         self.assertEqual({'rev1': ('null:',)}, graph.get_parent_map(['rev1']))
 
 
+class TestRepositoryGetRevisions(TestRemoteRepository):
+
+    def test_hpss_missing_revision(self):
+        transport_path = 'quack'
+        repo, client = self.setup_fake_client_and_repository(transport_path)
+        client.add_success_response_with_body(
+            '', 'ok', '10')
+        self.assertRaises(errors.NoSuchRevision, repo.get_revisions,
+            ['somerev1', 'anotherrev2'])
+        self.assertEqual(
+            [('call_with_body_bytes_expecting_body', 'Repository.iter_revisions',
+             ('quack/', ), "somerev1\nanotherrev2")],
+            client._calls)
+
+    def test_hpss_get_single_revision(self):
+        transport_path = 'quack'
+        repo, client = self.setup_fake_client_and_repository(transport_path)
+        somerev1 = Revision("somerev1")
+        somerev1.committer = "Joe Committer <joe at example.com>"
+        somerev1.timestamp = 1321828927
+        somerev1.timezone = -60
+        somerev1.inventory_sha1 = "691b39be74c67b1212a75fcb19c433aaed903c2b"
+        somerev1.message = "Message"
+        body = zlib.compress(chk_bencode_serializer.write_revision_to_string(
+            somerev1))
+        # Split up body into two bits to make sure the zlib compression object
+        # gets data fed twice.
+        client.add_success_response_with_body(
+                [body[:10], body[10:]], 'ok', '10')
+        revs = repo.get_revisions(['somerev1'])
+        self.assertEquals(revs, [somerev1])
+        self.assertEqual(
+            [('call_with_body_bytes_expecting_body', 'Repository.iter_revisions',
+             ('quack/', ), "somerev1")],
+            client._calls)
+
+
 class TestRepositoryGetRevisionGraph(TestRemoteRepository):
 
     def test_null_revision(self):

=== modified file 'bzrlib/tests/test_smart.py'
--- a/bzrlib/tests/test_smart.py	2011-11-28 03:38:43 +0000
+++ b/bzrlib/tests/test_smart.py	2011-11-28 11:34:18 +0000
@@ -25,6 +25,7 @@
 """
 
 import bz2
+import zlib
 
 from bzrlib import (
     branch as _mod_branch,
@@ -1724,6 +1725,50 @@
             request.execute('stacked', 1, (3, r3)))
 
 
+class TestSmartServerRepositoryIterRevisions(
+    tests.TestCaseWithMemoryTransport):
+
+    def test_basic(self):
+        backing = self.get_transport()
+        request = smart_repo.SmartServerRepositoryIterRevisions(backing)
+        tree = self.make_branch_and_memory_tree('.', format='2a')
+        tree.lock_write()
+        tree.add('')
+        tree.commit('1st commit', rev_id="rev1")
+        tree.commit('2nd commit', rev_id="rev2")
+        tree.unlock()
+
+        self.assertIs(None, request.execute(''))
+        response = request.do_body("rev1\nrev2")
+        self.assertTrue(response.is_successful())
+        # Format 2a uses serializer format 10
+        self.assertEquals(response.args, ("ok", "10"))
+
+        self.addCleanup(tree.branch.lock_read().unlock)
+        entries = [zlib.compress(record.get_bytes_as("fulltext")) for record in
+            tree.branch.repository.revisions.get_record_stream(
+            [("rev1", ), ("rev2", )], "unordered", True)]
+
+        contents = "".join(response.body_stream)
+        self.assertTrue(contents in (
+            "".join([entries[0], entries[1]]),
+            "".join([entries[1], entries[0]])))
+
+    def test_missing(self):
+        backing = self.get_transport()
+        request = smart_repo.SmartServerRepositoryIterRevisions(backing)
+        tree = self.make_branch_and_memory_tree('.', format='2a')
+
+        self.assertIs(None, request.execute(''))
+        response = request.do_body("rev1\nrev2")
+        self.assertTrue(response.is_successful())
+        # Format 2a uses serializer format 10
+        self.assertEquals(response.args, ("ok", "10"))
+
+        contents = "".join(response.body_stream)
+        self.assertEquals(contents, "")
+
+
 class GetStreamTestBase(tests.TestCaseWithMemoryTransport):
 
     def make_two_commit_repo(self):
@@ -2390,6 +2435,8 @@
             smart_repo.SmartServerRepositoryGetStream)
         self.assertHandlerEqual('Repository.get_stream_1.19',
             smart_repo.SmartServerRepositoryGetStream_1_19)
+        self.assertHandlerEqual('Repository.iter_revisions',
+            smart_repo.SmartServerRepositoryIterRevisions)
         self.assertHandlerEqual('Repository.has_revision',
             smart_repo.SmartServerRequestHasRevision)
         self.assertHandlerEqual('Repository.insert_stream',

=== modified file 'doc/en/release-notes/bzr-2.5.txt'
--- a/doc/en/release-notes/bzr-2.5.txt	2011-11-28 11:49:43 +0000
+++ b/doc/en/release-notes/bzr-2.5.txt	2011-11-28 13:05:15 +0000
@@ -128,7 +128,7 @@
   ``VersionedFileRepository.get_serializer_format``,
   ``Repository.all_revision_ids``, ``Repository.start_write_group``,
   ``Repository.commit_write_group``, ``Repository.abort_write_group``
-  ``Repository.check_write_group`` and
+  ``Repository.iter_revisions``, ``Repository.check_write_group`` and
   ``Repository.add_signature_revision_text``.
   (Jelmer Vernooij)
 




More information about the bazaar-commits mailing list