Rev 4174: Check that setting _combine_spilled_indices has the expected effect. in lp:///~jameinel/bzr/1.14-btree_spill
John Arbash Meinel
john at arbash-meinel.com
Mon Mar 23 19:21:05 GMT 2009
At lp:///~jameinel/bzr/1.14-btree_spill
------------------------------------------------------------
revno: 4174
revision-id: john at arbash-meinel.com-20090323192057-eh1l34z1ab5x3qt4
parent: john at arbash-meinel.com-20090321024706-drh9950i56vb484i
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 1.14-btree_spill
timestamp: Mon 2009-03-23 14:20:57 -0500
message:
Check that setting _combine_spilled_indices has the expected effect.
-------------- next part --------------
=== modified file 'bzrlib/btree_index.py'
--- a/bzrlib/btree_index.py 2009-03-21 02:47:06 +0000
+++ b/bzrlib/btree_index.py 2009-03-23 19:20:57 +0000
@@ -181,6 +181,34 @@
combine mem with the first and second indexes, creating a new one of
size 4x. On the fifth create a single new one, etc.
"""
+ if self._combine_spilled_indices:
+ (new_backing_file, size,
+ backing_pos) = self._spill_mem_keys_and_combine()
+ else:
+ new_backing_file, size = self._spill_mem_keys_without_combining()
+ dir_path, base_name = osutils.split(new_backing_file.name)
+ # Note: The transport here isn't strictly needed, because we will use
+ # direct access to the new_backing._file object
+ new_backing = BTreeGraphIndex(get_transport(dir_path),
+ base_name, size)
+ # GC will clean up the file
+ new_backing._file = new_backing_file
+ if self._combine_spilled_indices:
+ if len(self._backing_indices) == backing_pos:
+ self._backing_indices.append(None)
+ self._backing_indices[backing_pos] = new_backing
+ for backing_pos in range(backing_pos):
+ self._backing_indices[backing_pos] = None
+ else:
+ self._backing_indices.append(new_backing)
+ self._keys = set()
+ self._nodes = {}
+ self._nodes_by_key = None
+
+ def _spill_mem_keys_without_combining(self):
+ return self._write_nodes(self._iter_mem_nodes(), allow_optimize=False)
+
+ def _spill_mem_keys_and_combine(self):
iterators_to_combine = [self._iter_mem_nodes()]
pos = -1
for pos, backing in enumerate(self._backing_indices):
@@ -192,21 +220,7 @@
new_backing_file, size = \
self._write_nodes(self._iter_smallest(iterators_to_combine),
allow_optimize=False)
- dir_path, base_name = osutils.split(new_backing_file.name)
- # Note: The transport here isn't strictly needed, because we will use
- # direct access to the new_backing._file object
- new_backing = BTreeGraphIndex(get_transport(dir_path),
- base_name, size)
- # GC will clean up the file
- new_backing._file = new_backing_file
- if len(self._backing_indices) == backing_pos:
- self._backing_indices.append(None)
- self._backing_indices[backing_pos] = new_backing
- for pos in range(backing_pos):
- self._backing_indices[pos] = None
- self._keys = set()
- self._nodes = {}
- self._nodes_by_key = None
+ return new_backing_file, size, backing_pos
def add_nodes(self, nodes):
"""Add nodes to the index.
=== modified file 'bzrlib/tests/test_btree_index.py'
--- a/bzrlib/tests/test_btree_index.py 2009-03-21 02:47:06 +0000
+++ b/bzrlib/tests/test_btree_index.py 2009-03-23 19:20:57 +0000
@@ -458,38 +458,28 @@
self.assertEqual(0, len(builder._keys))
self.assertIs(None, builder._nodes_by_key)
self.assertEqual(2, len(builder._backing_indices))
- self.assertEqual(None, builder._backing_indices[0])
- self.assertEqual(4, builder._backing_indices[1].key_count())
- # The next spills to the 2-len slot
+ for backing_index in builder._backing_indices:
+ self.assertEqual(2, backing_index.key_count())
+ # The next spills to the 3rd slot
builder.add_node(*nodes[4])
builder.add_node(*nodes[5])
self.assertEqual(0, len(builder._nodes))
self.assertEqual(0, len(builder._keys))
self.assertIs(None, builder._nodes_by_key)
- self.assertEqual(2, len(builder._backing_indices))
- self.assertEqual(2, builder._backing_indices[0].key_count())
- self.assertEqual(4, builder._backing_indices[1].key_count())
- # Next spill combines
+ self.assertEqual(3, len(builder._backing_indices))
+ for backing_index in builder._backing_indices:
+ self.assertEqual(2, backing_index.key_count())
+ # Now spill a few more, and check that we don't combine
builder.add_node(*nodes[6])
builder.add_node(*nodes[7])
- self.assertEqual(3, len(builder._backing_indices))
- self.assertEqual(None, builder._backing_indices[0])
- self.assertEqual(None, builder._backing_indices[1])
- self.assertEqual(8, builder._backing_indices[2].key_count())
- # And so forth - counting up in binary.
builder.add_node(*nodes[8])
builder.add_node(*nodes[9])
- self.assertEqual(3, len(builder._backing_indices))
- self.assertEqual(2, builder._backing_indices[0].key_count())
- self.assertEqual(None, builder._backing_indices[1])
- self.assertEqual(8, builder._backing_indices[2].key_count())
builder.add_node(*nodes[10])
builder.add_node(*nodes[11])
- self.assertEqual(3, len(builder._backing_indices))
- self.assertEqual(None, builder._backing_indices[0])
- self.assertEqual(4, builder._backing_indices[1].key_count())
- self.assertEqual(8, builder._backing_indices[2].key_count())
builder.add_node(*nodes[12])
+ self.assertEqual(6, len(builder._backing_indices))
+ for backing_index in builder._backing_indices:
+ self.assertEqual(2, backing_index.key_count())
# Test that memory and disk are both used for query methods; and that
# None is skipped over happily.
self.assertEqual([(builder,) + node for node in sorted(nodes[:13])],
@@ -501,17 +491,11 @@
self.assertEqual(set([(builder,) + node for node in nodes[11:13]]),
set(builder.iter_entries_prefix([nodes[12][0], nodes[11][0]])))
builder.add_node(*nodes[13])
- self.assertEqual(3, len(builder._backing_indices))
- self.assertEqual(2, builder._backing_indices[0].key_count())
- self.assertEqual(4, builder._backing_indices[1].key_count())
- self.assertEqual(8, builder._backing_indices[2].key_count())
builder.add_node(*nodes[14])
builder.add_node(*nodes[15])
- self.assertEqual(4, len(builder._backing_indices))
- self.assertEqual(None, builder._backing_indices[0])
- self.assertEqual(None, builder._backing_indices[1])
- self.assertEqual(None, builder._backing_indices[2])
- self.assertEqual(16, builder._backing_indices[3].key_count())
+ self.assertEqual(8, len(builder._backing_indices))
+ for backing_index in builder._backing_indices:
+ self.assertEqual(2, backing_index.key_count())
# Now finish, and check we got a correctly ordered tree
transport = self.get_transport('')
size = transport.put_file('index', builder.finish())
More information about the bazaar-commits
mailing list