Rev 29: Search for revisions to index incrementally rather than reindexing everything every time. in http://people.ubuntu.com/~robertc/baz2.0/plugins/search/trunk

[Robert Collins]

At http://people.ubuntu.com/~robertc/baz2.0/plugins/search/trunk

------------------------------------------------------------
revno: 29
revision-id: robertc at robertcollins.net-20080613041235-4qi8o7xj26c2g77t
parent: robertc at robertcollins.net-20080612233548-flhaytd97uyvuzpf
committer: Robert Collins <robertc at robertcollins.net>
branch nick: trunk
timestamp: Fri 2008-06-13 14:12:35 +1000
message:
  Search for revisions to index incrementally rather than reindexing everything every time.
modified:
  index.py                       index.py-20080608055509-hnimeek7q8tctkqf-2
  tests/test_index.py            test_index.py-20080608055509-hnimeek7q8tctkqf-4
=== modified file 'index.py'
--- a/index.py	2008-06-12 23:35:48 +0000
+++ b/index.py	2008-06-13 04:12:35 +0000
@@ -84,19 +84,28 @@
         searcher = graph._make_breadth_first_searcher([_last_revid])
         try:
             index = open_index_url(url)
-            # TODO: find the unindexed revisions by searching.
+            index._refresh_indices()
+            revision_index = index._revision_index
+            while True:
+                try:
+                    next_revs, ghosts = searcher.next_with_ghosts()
+                except StopIteration:
+                    break
+                else:
+                    rev_keys = [(rev,) for rev in next_revs]
+                    indexed_revs = set([node[1][0] for node in
+                        revision_index.iter_entries(rev_keys)])
+                    unindexed_revs = next_revs - indexed_revs
+                revs_to_index.update(unindexed_revs)
         except errors.NoSearchIndex:
             index = init_index(branch)
-
-        while True:
-            try:
-                next_revs, ghosts = searcher.next_with_ghosts()
-            except StopIteration:
-                break
-            revs_to_index.update(next_revs)
-
+            while True:
+                try:
+                    next_revs, ghosts = searcher.next_with_ghosts()
+                except StopIteration:
+                    break
+                revs_to_index.update(next_revs)
         revs_to_index.remove(NULL_REVISION)
-            
         index.index_revisions(branch, revs_to_index)
     finally:
         branch.unlock()
@@ -286,7 +295,7 @@
         if max_components < 1:
             max_components = 1
         excess = len(self._current_names) - max_components
-        if not excess:
+        if excess < 1:
             return
         old_components = []
         for name, (value, component) in self._current_names.iteritems():

=== modified file 'tests/test_index.py'
--- a/tests/test_index.py	2008-06-12 23:35:48 +0000
+++ b/tests/test_index.py	2008-06-13 04:12:35 +0000
@@ -81,6 +81,29 @@
         rev_index = index.index_url(self.get_url('foo'))
         self.assertEqual(set([(revid,)]), set(rev_index.indexed_revisions()))
 
+    def test_index_url_is_incremental(self):
+        tree = self.make_branch_and_tree('foo')
+        # two initial commits (as we want to avoid the first autopack)
+        revid1 = tree.commit('1')
+        revid2 = tree.commit('2')
+        rev_index = index.index_url(self.get_url('foo'))
+        self.assertEqual(set([(revid1,), (revid2,)]),
+            set(rev_index.indexed_revisions()))
+        base_names = rev_index._current_names.keys()
+        self.assertEqual(1, len(base_names))
+        revid3 = tree.commit('3')
+        rev_index = index.index_url(self.get_url('foo'))
+        self.assertEqual(set([(revid1,), (revid2,), (revid3,)]),
+            set(rev_index.indexed_revisions()))
+        new_names = rev_index._current_names.keys()
+        self.assertSubset(base_names, new_names)
+        self.assertEqual(2, len(new_names))
+        # The new index should only have revid3 in it.
+        new_name = list(set(new_names) - set(base_names))[0]
+        new_component = rev_index._current_names[new_name][1]
+        self.assertEqual([(revid3,)], [node[1] for node in
+            new_component.revision_index.iter_all_entries()])
+
     def test_index_combining(self):
         # After inserting 1 revision, we get one pack,
         # After 2 we should still have 1, but also two discards