Rev 16: Allow lsprofiling of individual fixtures, and start applying the results generated from that analysis. in http://people.ubuntu.com/~robertc/baz2.0/plugins/index2/trunk

[Robert Collins]

At http://people.ubuntu.com/~robertc/baz2.0/plugins/index2/trunk

------------------------------------------------------------
revno: 16
revision-id: robertc at robertcollins.net-20080702125159-o72w19dm8g5yrtt0
parent: robertc at robertcollins.net-20080702093116-q5iu1k5bpd7icqlk
committer: Robert Collins <robertc at robertcollins.net>
branch nick: trunk
timestamp: Wed 2008-07-02 22:51:59 +1000
message:
  Allow lsprofiling of individual fixtures, and start applying the results generated from that analysis.
modified:
  btree_index.py                 index.py-20080624222253-p0x5f92uyh5hw734-7
  indexbench.py                  indexbench.py-20080702083855-5tju02y79rw7kkzh-1
=== modified file 'btree_index.py'
--- a/btree_index.py	2008-07-02 08:43:38 +0000
+++ b/btree_index.py	2008-07-02 12:51:59 +0000
@@ -425,13 +425,17 @@
             keys supplied. No additional keys will be returned, and every
             key supplied that is in the index will be returned.
         """
-        keys = sorted(keys)
+        if self._key_count is None:
+            self._cache_nodes([0])
+        if not self._key_count:
+            return
+        # 6 seconds spent in miss_torture using the sorted() line.
+        # Even with out of order disk IO it seems faster not to sort it when
+        # large queries are being made.
+        # keys = sorted(keys)
+        keys = frozenset(keys)
         if not keys:
             return
-        if self._key_count is None:
-            self._cache_nodes([0])
-        if not self._key_count:
-            return
         for key in keys:
             # repeated single-bisection : 'make it work'
             # FUTURE: look within the current node for the next key,

=== modified file 'indexbench.py'
--- a/indexbench.py	2008-07-02 09:31:16 +0000
+++ b/indexbench.py	2008-07-02 12:51:59 +0000
@@ -13,6 +13,7 @@
 from bzrlib.graph import Graph
 from bzrlib.commands import Command
 from bzrlib.option import Option, ListOption
+from bzrlib.lsprof import profile
 
 
 key_count = 0
@@ -39,12 +40,28 @@
     return size
 
 
+def profile_fixture(class_name, fixture_name, fixture, *args, **kwargs):
+    """Profile a fixture."""
+    _, stats = profile(fixture, *args, **kwargs)
+    output = file(class_name + '.' + fixture_name + '.callgrind', 'wb')
+    stats.calltree(output)
+    output.close()
+
+
+def run_fixture(class_name, fixture_name, fixture, *args, **kwargs):
+    fixture(*args, **kwargs)
+
+
 def time_index(names, source, factory, builder, target, label, name_keys,
-    text_keys, use_drop_cache, fixtures):
+    text_keys, use_drop_cache, fixtures, lsprof):
     if use_drop_cache:
         drop_cache = drop_caches
     else:
         drop_cache = lambda:None
+    if lsprof:
+        run = profile_fixture
+    else:
+        run = run_fixture
     drop_cache()
 # Baseline time
     now = time.time()
@@ -127,6 +144,10 @@
 # pathologic miss-lookups: ask each index of each type for the keys held by the
 # others of that type
     if 'miss' in fixtures:
+        run(label, 'miss', miss_torture, label, drop_cache, names, target, factory, name_keys)
+    print "%s: -------Done---------" % (label,)
+
+def miss_torture(label, drop_cache, names, target, factory, name_keys):
         btree_index.bloom_hits = 0
         drop_cache()
         now = time.time()
@@ -140,7 +161,6 @@
                 pass
         finish = time.time()
         print "%s: miss torture in %0.3f, bloom(%d)" % (label, finish - now, btree_index.bloom_hits)
-        print "%s: -------Done---------" % (label,)
 
 
 class cmd_indexbench(Command):
@@ -151,14 +171,16 @@
         Option('btree', help='bench the BTreeGraphIndex class'),
         Option('drop-cache', help='drop caches between fixtures'),
         ListOption('fixture', type=str,
-            help='fixtures to test: one of all, shuffle, text, revision, miss')
+            help='fixtures to test: one of all, shuffle, text, revision, miss'),
+        # lspro because --lsprof is a global option, and they interfere with each other.
+        Option('lspro', help='generate class.fixture.callgrind lsprof files'),
         ]
 
 
     takes_args = ['sample_repository']
 
     def run(self, sample_repository, graph=True, btree=True, drop_cache=False,
-        fixture=None):
+        fixture=None,lspro=False):
         if not fixture:
             fixture = ['all', 'shuffle', 'text', 'revision', 'miss']
         source = get_transport(sample_repository)
@@ -182,17 +204,17 @@
 
         if btree:
             self.test_class(names, source, BTreeGraphIndex, BTreeBuilder,
-                'BTreeIndex', name_keys, text_keys, drop_cache, fixture)
+                'BTreeIndex', name_keys, text_keys, drop_cache, fixture, lspro)
         if graph:
             self.test_class(names, source, GraphIndex, GraphIndexBuilder,
-                'GraphIndex', name_keys, text_keys, drop_cache, fixture)
+                'GraphIndex', name_keys, text_keys, drop_cache, fixture, lsprof)
 
     def test_class(self, names, source, factory, builder, label, name_keys,
-        text_keys, drop_cache, fixtures):
+        text_keys, drop_cache, fixtures, lsprof):
         workingdir = tempfile.mkdtemp()
         t = get_transport(workingdir)
         try:
             time_index(names, source, factory, builder, t, label, name_keys,
-                text_keys, drop_cache, fixtures)
+                text_keys, drop_cache, fixtures, lsprof)
         finally:
             t.delete_tree('.')