Rev 3871: Batch revision retrieval in InterDifferingSerializer to reduce the in file:///home/pqm/archives/thelove/bzr/%2Btrunk/
Canonical.com Patch Queue Manager
pqm at pqm.ubuntu.com
Fri Nov 28 03:51:44 GMT 2008
At file:///home/pqm/archives/thelove/bzr/%2Btrunk/
------------------------------------------------------------
revno: 3871
revision-id: pqm at pqm.ubuntu.com-20081128035141-nuniuees80e1d2ic
parent: pqm at pqm.ubuntu.com-20081128031739-mprtvyhcn9zwtkqc
parent: andrew.bennetts at canonical.com-20081128023559-tr1gp67zh2u0ib37
committer: Canonical.com Patch Queue Manager <pqm at pqm.ubuntu.com>
branch nick: +trunk
timestamp: Fri 2008-11-28 03:51:41 +0000
message:
Batch revision retrieval in InterDifferingSerializer to reduce the
number of network round trips.
modified:
NEWS NEWS-20050323055033-4e00b5db738777ff
bzrlib/repository.py rev_storage.py-20051111201905-119e9401e46257e3
------------------------------------------------------------
revno: 3831.2.3
revision-id: andrew.bennetts at canonical.com-20081128023559-tr1gp67zh2u0ib37
parent: andrew.bennetts at canonical.com-20081128023304-ffcfuwrau3nswe78
committer: Andrew Bennetts <andrew.bennetts at canonical.com>
branch nick: faster-differing-serialiser
timestamp: Fri 2008-11-28 13:35:59 +1100
message:
Add NEWS entry.
modified:
NEWS NEWS-20050323055033-4e00b5db738777ff
------------------------------------------------------------
revno: 3831.2.2
revision-id: andrew.bennetts at canonical.com-20081128023304-ffcfuwrau3nswe78
parent: andrew.bennetts at canonical.com-20081112070240-48b8da96jutbzzgl
parent: pqm at pqm.ubuntu.com-20081128021914-ajryb98qjwow6wfv
committer: Andrew Bennetts <andrew.bennetts at canonical.com>
branch nick: faster-differing-serialiser
timestamp: Fri 2008-11-28 13:33:04 +1100
message:
Merge bzr.dev.
added:
bzrlib/foreign.py foreign.py-20081112170002-olsxmandkk8qyfuq-1
bzrlib/tests/test_foreign.py test_foreign.py-20081125004048-ywb901edgp9lluxo-1
contrib/convert_to_1.9.py convert_to_dev2.py-20081014130524-z1ydl3mq9b4ehlvv-1
tools/win32/build_release.py build_release.py-20081105204355-2ghh5cv01v1x4rzz-1
modified:
NEWS NEWS-20050323055033-4e00b5db738777ff
bzrlib/__init__.py __init__.py-20050309040759-33e65acf91bbcd5d
bzrlib/_readdir_pyx.pyx readdir.pyx-20060609152855-rm6v321vuaqyh9tu-1
bzrlib/branch.py branch.py-20050309040759-e4baf4e0d046576e
bzrlib/btree_index.py index.py-20080624222253-p0x5f92uyh5hw734-7
bzrlib/bugtracker.py bugtracker.py-20070410073305-vu1vu1qosjurg8kb-1
bzrlib/builtins.py builtins.py-20050830033751-fc01482b9ca23183
bzrlib/commit.py commit.py-20050511101309-79ec1a0168e0e825
bzrlib/config.py config.py-20051011043216-070c74f4e9e338e8
bzrlib/errors.py errors.py-20050309040759-20512168c4e14fbd
bzrlib/fetch.py fetch.py-20050818234941-26fea6105696365d
bzrlib/help_topics/__init__.py help_topics.py-20060920210027-rnim90q9e0bwxvy4-1
bzrlib/index.py index.py-20070712131115-lolkarso50vjr64s-1
bzrlib/knit.py knit.py-20051212171256-f056ac8f0fbe1bd9
bzrlib/lockable_files.py control_files.py-20051111201905-bb88546e799d669f
bzrlib/log.py log.py-20050505065812-c40ce11702fe5fb1
bzrlib/mutabletree.py mutabletree.py-20060906023413-4wlkalbdpsxi2r4y-2
bzrlib/option.py option.py-20051014052914-661fb36e76e7362f
bzrlib/plugin.py plugin.py-20050622060424-829b654519533d69
bzrlib/plugins/launchpad/account.py account.py-20071011033320-50y6vfftywf4yllw-1
bzrlib/plugins/launchpad/lp_directory.py lp_indirect.py-20070126012204-de5rugwlt22c7u7e-1
bzrlib/plugins/launchpad/test_account.py test_account.py-20071011033320-50y6vfftywf4yllw-2
bzrlib/python-compat.h pythoncompat.h-20080924041409-9kvi0fgtuuqp743j-1
bzrlib/remote.py remote.py-20060720103555-yeeg2x51vn0rbtdp-1
bzrlib/repofmt/pack_repo.py pack_repo.py-20070813041115-gjv5ma7ktfqwsjgn-1
bzrlib/repofmt/weaverepo.py presplitout.py-20070125045333-wfav3tsh73oxu3zk-1
bzrlib/repository.py rev_storage.py-20051111201905-119e9401e46257e3
bzrlib/shelf_ui.py shelver.py-20081005210102-33worgzwrtdw0yrm-1
bzrlib/tests/__init__.py selftest.py-20050531073622-8d0e3c8845c97a64
bzrlib/tests/blackbox/test_log.py test_log.py-20060112090212-78f6ea560c868e24
bzrlib/tests/branch_implementations/test_sprout.py test_sprout.py-20070521151739-b8t8p7axw1h966ws-1
bzrlib/tests/branch_implementations/test_stacking.py test_stacking.py-20080214020755-msjlkb7urobwly0f-1
bzrlib/tests/commands/test_commit.py test_commit.py-20070913161801-ydrx2k5gmv7k7eiu-1
bzrlib/tests/interrepository_implementations/test_fetch.py test_fetch.py-20080425213627-j60cjh782ufm83ry-1
bzrlib/tests/per_repository/test_commit_builder.py test_commit_builder.py-20060606110838-76e3ra5slucqus81-1
bzrlib/tests/per_repository/test_repository.py test_repository.py-20060131092128-ad07f494f5c9d26c
bzrlib/tests/per_repository/test_revision.py testrevprops.py-20051013073044-92bc3c68302ce1bf
bzrlib/tests/per_repository/test_write_group.py test_write_group.py-20070716105516-89n34xtogq5frn0m-1
bzrlib/tests/test_btree_index.py test_index.py-20080624222253-p0x5f92uyh5hw734-13
bzrlib/tests/test_fetch.py testfetch.py-20050825090644-f73e07e7dfb1765a
bzrlib/tests/test_knit.py test_knit.py-20051212171302-95d4c00dd5f11f2b
bzrlib/tests/test_log.py testlog.py-20050728115707-1a514809d7d49309
bzrlib/tests/test_pack_repository.py test_pack_repository-20080801043947-eaw0e6h2gu75kwmy-1
bzrlib/tests/test_permissions.py test_permissions.py-20051215004520-ccf475789c80e80c
bzrlib/tests/test_plugins.py plugins.py-20050622075746-32002b55e5e943e9
bzrlib/tests/test_remote.py test_remote.py-20060720103555-yeeg2x51vn0rbtdp-2
bzrlib/tests/test_repository.py test_repository.py-20060131075918-65c555b881612f4d
bzrlib/tests/test_revision.py testrevision.py-20050804210559-46f5e1eb67b01289
bzrlib/tests/test_sftp_transport.py testsftp.py-20051027032739-247570325fec7e7e
bzrlib/tests/test_shelf_ui.py test_shelf_ui.py-20081027155203-wtcuazg85wp9u4fv-1
bzrlib/tests/workingtree_implementations/test_parents.py test_set_parents.py-20060807231740-yicmnlci1mj8smu1-1
bzrlib/transport/remote.py ssh.py-20060608202016-c25gvf1ob7ypbus6-1
bzrlib/versionedfile.py versionedfile.py-20060222045106-5039c71ee3b65490
bzrlib/workingtree.py workingtree.py-20050511021032-29b6ec0a681e02e3
bzrlib/workingtree_4.py workingtree_4.py-20070208044105-5fgpc5j3ljlh5q6c-1
------------------------------------------------------------
revno: 3831.2.1
revision-id: andrew.bennetts at canonical.com-20081112070240-48b8da96jutbzzgl
parent: pqm at pqm.ubuntu.com-20081112012514-6y8u99lf11pk0rdm
committer: Andrew Bennetts <andrew.bennetts at canonical.com>
branch nick: faster-differing-serialiser
timestamp: Wed 2008-11-12 17:02:40 +1000
message:
Quick hack to do batching in InterDifferingSerializer. Almost halves the HPSS round-trips fetching pack-0.92-subtree to 1.9-rich-root.
modified:
bzrlib/repository.py rev_storage.py-20051111201905-119e9401e46257e3
=== modified file 'NEWS'
--- a/NEWS 2008-11-28 01:43:35 +0000
+++ b/NEWS 2008-11-28 02:35:59 +0000
@@ -24,6 +24,11 @@
history just to set the last revision info.
(Andrew Bennetts, John Arbash Meinel)
+ * Fetches between formats with different serializers (such as
+ pack-0.92-subtree and 1.9-rich-root) are faster now. This is due to
+ operating on batches of 100 revisions at time rather than
+ one-by-one. (Andrew Bennetts, John Arbash Meinel)
+
* Search index files corresponding to pack files we've already used
before searching others, because they are more likely to have the
keys we're looking for. This reduces the number of iix and tix
=== modified file 'bzrlib/repository.py'
--- a/bzrlib/repository.py 2008-11-28 01:42:05 +0000
+++ b/bzrlib/repository.py 2008-11-28 02:33:04 +0000
@@ -38,6 +38,7 @@
symbol_versioning,
tsort,
ui,
+ versionedfile,
)
from bzrlib.bundle import serializer
from bzrlib.revisiontree import RevisionTree
@@ -3090,15 +3091,19 @@
revision_ids = tsort.topo_sort(
self.source.get_graph().get_parent_map(revision_ids))
def revisions_iterator():
- for current_revision_id in revision_ids:
- revision = self.source.get_revision(current_revision_id)
- tree = self.source.revision_tree(current_revision_id)
- try:
- signature = self.source.get_signature_text(
- current_revision_id)
- except errors.NoSuchRevision:
- signature = None
- yield revision, tree, signature
+ rev_ids = list(revision_ids)
+ for offset in xrange(0, len(rev_ids), 100):
+ current_revids = rev_ids[offset:offset+100]
+ revisions = self.source.get_revisions(current_revids)
+ trees = self.source.revision_trees(current_revids)
+ keys = [(r,) for r in current_revids]
+ sig_stream = self.source.signatures.get_record_stream(
+ keys, 'unordered', True)
+ sigs = {}
+ for record in versionedfile.filter_absent(sig_stream):
+ sigs[record.key[0]] = record.get_bytes_as('fulltext')
+ for rev, tree in zip(revisions, trees):
+ yield rev, tree, sigs.get(rev.revision_id, None)
if pb is None:
my_pb = ui.ui_factory.nested_progress_bar()
pb = my_pb
More information about the bazaar-commits
mailing list