[MERGE] Add --coverage option to selftest

Andrew Bennetts andrew at canonical.com
Thu Dec 6 12:13:48 GMT 2007


Hi all,

I've been thinking selftest ought to have a --coverage option for some time.
Tonight I finally sat down and wrote it.  It simply uses the trace module from
the standard library in a pretty simple way (basically the same way that
Twisted's test runner uses it).  It causes a fair bit of noise on stdout (or
stderr?) while writing the output at the end of the test run.  Like the
profiler, it uses sys.settrace so it make the test suite much slower when active
(and probably doesn't work at the same time as the profiler for that matter).
Despite the rough corners, I think this is useful enough to merge as is.

The basic idea is you say “--coverage=/tmp/coverage-output” and then
/tmp/coverage-output will have a file per module with the source of that module
annotate with the number of times a line was executed.  If a line was never
executed, it is highlighted by “>>>>>>”.  Thus you can easily spot gaps in your
test coverage.  (While the fact that a line *is* executed doesn't necessarily
mean it is adequately tested, clearly any line that is never executed isn't,
except in very unusual circumstances).

-Andrew.

-------------- next part --------------
# Bazaar merge directive format 2 (Bazaar 0.90)
# revision_id: andrew.bennetts at canonical.com-20071206120335-\
#   o7s9lfxyupf08nfm
# target_branch: http://bazaar-vcs.org/bzr/bzr.dev
# testament_sha1: 6180877ebca8850a01fd17b596886a08c1e8bcb9
# timestamp: 2007-12-06 23:04:34 +1100
# source_branch: http://people.ubuntu.com/~andrew/bzr/selftest-\
#   coverage
# base_revision_id: pqm at pqm.ubuntu.com-20071205233104-ghmsse3nmhbnz0n1
# 
# Begin patch
=== modified file 'NEWS'
--- NEWS	2007-12-05 22:52:58 +0000
+++ NEWS	2007-12-06 12:03:35 +0000
@@ -9,6 +9,8 @@
 
   IMPROVEMENTS:
 
+   * Added a --coverage option to selftest. (Andrew Bennetts)
+
    * ``bzr commit`` now doesn't print the revision number twice. (Matt
      Nordhoff, #172612)
 

=== modified file 'bzrlib/builtins.py'
--- bzrlib/builtins.py	2007-12-05 23:31:04 +0000
+++ bzrlib/builtins.py	2007-12-06 12:03:35 +0000
@@ -2604,6 +2604,9 @@
                                  ' expression.'),
                      Option('strict', help='Fail on missing dependencies or '
                             'known failures.'),
+                     Option('coverage', type=str, argname="DIRECTORY",
+                            help='Generate line coverage report in this'
+                                 'directory.'),
                      ]
     encoding_type = 'replace'
 
@@ -2611,7 +2614,7 @@
             transport=None, benchmark=None,
             lsprof_timed=None, cache_dir=None,
             first=False, list_only=False,
-            randomize=None, exclude=None, strict=False):
+            randomize=None, exclude=None, strict=False, coverage=None):
         import bzrlib.ui
         from bzrlib.tests import selftest
         import bzrlib.benchmarks as benchmarks
@@ -2653,6 +2656,7 @@
                               random_seed=randomize,
                               exclude_pattern=exclude,
                               strict=strict,
+                              coverage_dir=coverage,
                               )
         finally:
             if benchfile is not None:

=== modified file 'bzrlib/tests/__init__.py'
--- bzrlib/tests/__init__.py	2007-12-04 00:42:43 +0000
+++ bzrlib/tests/__init__.py	2007-12-06 12:03:35 +0000
@@ -42,8 +42,9 @@
 from subprocess import Popen, PIPE
 import sys
 import tempfile
+import time
+import trace
 import unittest
-import time
 import warnings
 
 
@@ -2286,6 +2287,7 @@
               random_seed=None,
               exclude_pattern=None,
               strict=False,
+              coverage_dir=None,
               ):
     TestCase._gather_lsprof_in_benchmarks = lsprof_timed
     if verbose:
@@ -2323,8 +2325,19 @@
         else:
             suite = filter_suite_by_re(suite, pattern, exclude_pattern,
                 random_order)
+
+    if coverage_dir is not None:
+        tracer = trace.Trace(count=1, trace=0)
+        sys.settrace(tracer.globaltrace)
+
     result = runner.run(suite)
 
+    if coverage_dir is not None:
+        sys.settrace(None)
+        results = tracer.results()
+        results.write_results(show_missing=1, summary=False,
+                              coverdir=coverage_dir)
+
     if strict:
         return result.wasStrictlySuccessful()
 
@@ -2341,6 +2354,7 @@
              random_seed=None,
              exclude_pattern=None,
              strict=False,
+             coverage_dir=None,
              ):
     """Run the whole test suite under the enhanced runner"""
     # XXX: Very ugly way to do this...
@@ -2368,7 +2382,8 @@
                      list_only=list_only,
                      random_seed=random_seed,
                      exclude_pattern=exclude_pattern,
-                     strict=strict)
+                     strict=strict,
+                     coverage_dir=coverage_dir)
     finally:
         default_transport = old_transport
 

# Begin bundle
IyBCYXphYXIgcmV2aXNpb24gYnVuZGxlIHY0CiMKQlpoOTFBWSZTWXZOhn0AA8j/gGRwIFJR////
f///8L////BgB04vpAHSincd9vUdADppoaERMEwpspmJkAUyHqZqaPKekZNp6p5TT9UHqG0aIEpN
Q0k9G1NpBqPUAAAAAGjRkAAAA5o0aGmEA0wJpoAyGhiANGI0MEZABpopk0mUGnplAaA9TQBoAAAA
AaDQBtUp+qQ21CPTSGjCZNMgDQyMTTQ0AZGmgABJEENCZBo0IaCap+mmp6GmqNqaMjagHqAD1PSb
UcFoEZFCU9wp6Jsb5yoDaPAexgbm/0WDusuYnkDxmsmXsRY47RR3b1IwCjHQdFalmfhJAAB5FJRJ
YCiDPWhnudO9nONFMkB1UrMECJAIAfnOmdF2ioDvSuhJRCUJAuegAhN9PrEHVUvE5ZbC4BE7LhmB
EppL3yMrlDBpYYacYU24thjwjzcBg0xA9YVUW4Jkt3kyhaKJvvu+ba1Qk3JNiQIwHZFO8qof7kK+
Ljj+mTN9+XR0cfs6dEeWsv10rsYSkdNHv3EAFQTKAIKyzehtjV8JWbAZBohnQPNLD3FnW1uQZzpv
I6ZSBroBAVFzk6O+6KQ8YuEQGakewX0eKtB1I4bflsbrBVaN9kbJbfWhfP8WZDAreXzqHIMZCbMP
lCmrF1D96G241oMWEZsjPZ6IVbyqaQrSw+soRKoqB6hyIK0zQ8oIHFEoVKcQTx4DjzSCRbAewFKt
yQSRKo95J6yVGJeR7/ojdPR/K/UBFn0PAYf2PnqyMD7tiLmPTpsK5RcSokFdu4sY7Sz4raWbQhGJ
Gm4YjCjXQ9mBecpVXQYP8TG2kjm2mAxcSTFsTYbO5IUixJKGOTEYNY1hoHIoYVkpOxGIRYIJ6ST3
ZWpTOcyLTTIc+cbtG26qWFpGBoMIokoDwHYZi4YcTCXByBEE+koPHukbGOHJBUJjBoZv4cY4cxJh
iFpa1lrHQWEKYFIgnqwIaW0znO2uqBAuMi82EiFZWQNhcXSYhYG5yI5PG+WRQFSe0xUxlFj8iLVe
fZhnMV9bjZMYFkmiLZGcj1e1lDyct0DBgoopQMG1q6rNCs6kVuT1XlVQ+gvIgoDM0DFjA1yGnjbe
OijlCxjEndSU3svrkMkDxMoFhY+sqKrSogU44GgVByJM6P0AteFFbpNejVqIKBcMaioseyZRyMBs
SrbgVZKSgppcCar0V7nExDAsuENtilhSdSwkcDFQj9HmWETWRtMy44HYC6enlP3ZnL2bdheHMbcG
DUKIoBrSl9R1+0vYD4MfLsI/iyqNHfAxbUnHGIplOG2OtbWKQEFCtXmA7FZyiYqzMgS2EuWqugWJ
5OC7NXQoZmDG0ZuloOp07Usq00vxP1HAbXMNEPgssGkbdUQ3lcWXZbcn5t/FsWUYuvv9fHhxv1Wg
4GjnZNI5CDQ4encyWxgvK9a5NhQ8/l8SOVwCPu4Tj35IpB48sXzw/TFDAXXit9uqvVj0EByA1IaL
NE3IV4KvDGJilU1okC8XDmLRmJdOEUUhIjaIBpd5kLgigTBEnAShF+2pmgLPWWsds0Z4ow8R2r7h
QGcj2VCds3OhmJ4L47Im1hYMVCkN0A5hXBy19Tj4zkCMdiOJxb2iawQesx1x/nDE07NaiYLI/jiD
7pGaFyQUoxsgr5uiNCalocWeiucAhh7GgnbrUokFG/xqN4rsByNyK+By5dDvM8TpLDG1FBMZl6Dk
H26BqHGnHnDzS7m4XgHp7PVD+q9f5lwveoJg/kOQurWJXX3D7DaOZWokGq2ZIuL/NPiicY3VwOOn
wZlADSmSh43FCxItQfDOOrziSX9gx0EbUPFGAbzJUdae1ESVku1bDILipJrQmVpJxCj1sBLZUZv2
lAWk0+Rjq4ntKFh9/iXB6D3mAf3Xeti2oTB1i9QwKBm3xoArke6Vuv8OZ2i7oOKLvw5+5Hqmb9Yl
JqatSE7MsfqAmf9PQFN6D1qStMz9iV/RGu2pfX/7nnSqaZIvEQPDvSN+zY9SbDfwPl92pLPpyYFR
LkggLtBX6Z7zonA6cSxQmGHMFGRyBH0E6YW2WU1ZYjS36uaJmjKu/Q1eYVHjwQ8mZHkjqdEd7hnv
BqfCxJ/eMhPNA3HSkvAFDKp5oUH8UVJKtCtIqGpkOwDi90A9qOf0FAXLm+nQCO1rEl8QPLJCqRiW
pXBv8wgEzK1AS5YUouvr6hbqg8vO4N4KUjGHwFX+lgkyU6BYuJECRWuS56qOWlGYTCZYnAwDwqfz
ZjopHhERu+mEX1eZJa2UOtJRrzusxXf16ghHaHsqkDDMdzqjVGpMuxdA/l0grIDMJQOsZHT89puh
1cDAcKLIR70Ej0zGANRwmxrvMNBqgVkS+S3oQWG5kj6TA4MeGoWG+onIwc/gKok5HX4iw8mFQa+8
dQXUnLUrr2HhQm5hveSnOaZnHQrHFfrQDixiVzRH56CK79omqWKJkVUhnvrSprEhJKEFvuAhgjth
39sEGAKq/cJ07bPGFpwBaEflmZoqty/+LuSKcKEg7J0M+g==


More information about the bazaar mailing list