Rev 3792: CombinedGraphIndex can now reload when calling key_count(). in http://bzr.arbash-meinel.com/branches/bzr/1.9-dev/pack_retry_153786

[John Arbash Meinel]

At http://bzr.arbash-meinel.com/branches/bzr/1.9-dev/pack_retry_153786

------------------------------------------------------------
revno: 3792
revision-id: john at arbash-meinel.com-20081023200801-72xbksw2uyj8cdmq
parent: john at arbash-meinel.com-20081023193959-xdytlck4ym0vr6lt
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: pack_retry_153786
timestamp: Thu 2008-10-23 15:08:01 -0500
message:
  CombinedGraphIndex can now reload when calling key_count().
-------------- next part --------------
=== modified file 'bzrlib/index.py'
--- a/bzrlib/index.py	2008-09-21 14:48:37 +0000
+++ b/bzrlib/index.py	2008-10-23 20:08:01 +0000
@@ -27,6 +27,7 @@
 from bisect import bisect_right
 from cStringIO import StringIO
 import re
+import sys
 
 from bzrlib.lazy_import import lazy_import
 lazy_import(globals(), """
@@ -1106,12 +1107,16 @@
     in the index list.
     """
 
-    def __init__(self, indices):
+    def __init__(self, indices, reload_func=None):
         """Create a CombinedGraphIndex backed by indices.
 
         :param indices: An ordered list of indices to query for data.
+        :param reload_func: A function to call if we find we are missing an
+            index. Should have the form reload_func() => True/False to indicate
+            if reloading actually changed anything.
         """
         self._indices = indices
+        self._reload_func = reload_func
 
     def __repr__(self):
         return "%s(%s)" % (
@@ -1227,13 +1232,22 @@
 
     def key_count(self):
         """Return an estimate of the number of keys in this index.
-        
+
         For CombinedGraphIndex this is approximated by the sum of the keys of
         the child indices. As child indices may have duplicate keys this can
         have a maximum error of the number of child indices * largest number of
         keys in any index.
         """
-        return sum((index.key_count() for index in self._indices), 0)
+        do_retry = True
+        while do_retry:
+            try:
+                return sum((index.key_count() for index in self._indices), 0)
+            except errors.NoSuchFile:
+                if self._reload_func is None:
+                    raise
+                exc_type, exc_value, exc_traceback = sys.exc_info()
+                if not self._reload_func():
+                    raise exc_type, exc_value, exc_traceback
 
     def validate(self):
         """Validate that everything in the index can be accessed."""

=== modified file 'bzrlib/repofmt/pack_repo.py'
--- a/bzrlib/repofmt/pack_repo.py	2008-10-23 19:39:59 +0000
+++ b/bzrlib/repofmt/pack_repo.py	2008-10-23 20:08:01 +0000
@@ -452,10 +452,16 @@
     # XXX: Probably 'can be written to' could/should be separated from 'acts
     # like a knit index' -- mbp 20071024
 
-    def __init__(self):
-        """Create an AggregateIndex."""
+    def __init__(self, reload_func=None):
+        """Create an AggregateIndex.
+
+        :param reload_func: A function to call if we find we are missing an
+            index. Should have the form reload_func() => True/False to indicate
+            if reloading actually changed anything.
+        """
+        self._reload_func = reload_func
         self.index_to_pack = {}
-        self.combined_index = CombinedGraphIndex([])
+        self.combined_index = CombinedGraphIndex([], reload_func=reload_func)
         self.data_access = _DirectPackAccess(self.index_to_pack)
         self.add_callback = None
 

=== modified file 'bzrlib/tests/test_index.py'
--- a/bzrlib/tests/test_index.py	2008-09-02 17:52:00 +0000
+++ b/bzrlib/tests/test_index.py	2008-10-23 20:08:01 +0000
@@ -927,6 +927,38 @@
         size = trans.put_file(name, stream)
         return GraphIndex(trans, name, size)
 
+    def make_index_with_missing_children(self):
+        """Create a CombinedGraphIndex which will have missing indexes.
+
+        This creates a CGI which thinks it has 2 indexes, however they have
+        been deleted. If CGI._reload_func() is called, then it will repopulate
+        with a new index.
+
+        :return: (CombinedGraphIndex, reload_counter)
+        """
+        index1 = self.make_index('1', nodes=[(('1',), '', ())])
+        index2 = self.make_index('2', nodes=[(('2',), '', ())])
+        index3 = self.make_index('3', nodes=[
+            (('1',), '', ()),
+            (('2',), '', ())])
+
+        # total_reloads, num_changed, num_unchanged
+        reload_counter = [0, 0, 0]
+        def reload():
+            reload_counter[0] += 1
+            new_indices = [index3]
+            if index._indices == new_indices:
+                reload_counter[2] += 1
+                return False
+            reload_counter[1] += 1
+            index._indices[:] = new_indices
+            return True
+        index = CombinedGraphIndex([index1, index2], reload_func=reload)
+        trans = self.get_transport()
+        trans.delete('1')
+        trans.delete('2')
+        return index, reload_counter
+
     def test_open_missing_index_no_error(self):
         trans = self.get_transport()
         index1 = GraphIndex(trans, 'missing', 100)
@@ -1070,6 +1102,20 @@
         index = CombinedGraphIndex([])
         index.validate()
 
+    def test_key_count_reloads(self):
+        index, reload_counter = self.make_index_with_missing_children()
+        self.assertEqual(2, index.key_count())
+        self.assertEqual([1, 1, 0], reload_counter)
+
+    def test_key_count_reloads_and_fails(self):
+        index, reload_counter = self.make_index_with_missing_children()
+        # We have deleted the underlying index, so we will try to reload, but
+        # still fail. This is mostly to test we don't get stuck in an infinite
+        # loop trying to reload
+        self.get_transport().delete('3')
+        self.assertRaises(errors.NoSuchFile, index.key_count)
+        self.assertEqual([2, 1, 1], reload_counter)
+
 
 class TestInMemoryGraphIndex(TestCaseWithMemoryTransport):