Rev 4637: Test selftest --list-only and --randomize options using more precisely layers. in http://bazaar.launchpad.net/~lifeless/bzr/test-speed
Robert Collins
robertc at robertcollins.net
Sun Aug 23 23:15:55 BST 2009
At http://bazaar.launchpad.net/~lifeless/bzr/test-speed
------------------------------------------------------------
revno: 4637
revision-id: robertc at robertcollins.net-20090823221551-rfquxm9tilojjeb0
parent: pqm at pqm.ubuntu.com-20090821071831-d3dacehbozpkpy27
committer: Robert Collins <robertc at robertcollins.net>
branch nick: test-speed
timestamp: Mon 2009-08-24 08:15:51 +1000
message:
Test selftest --list-only and --randomize options using more precisely layers.
=== modified file 'bzrlib/tests/__init__.py'
--- a/bzrlib/tests/__init__.py 2009-08-20 05:05:59 +0000
+++ b/bzrlib/tests/__init__.py 2009-08-23 22:15:51 +0000
@@ -3207,6 +3207,7 @@
starting_with=None,
runner_class=None,
suite_decorators=None,
+ stream=None,
):
"""Run the whole test suite under the enhanced runner"""
# XXX: Very ugly way to do this...
@@ -3245,6 +3246,7 @@
strict=strict,
runner_class=runner_class,
suite_decorators=suite_decorators,
+ stream=stream,
)
finally:
default_transport = old_transport
=== modified file 'bzrlib/tests/blackbox/test_selftest.py'
--- a/bzrlib/tests/blackbox/test_selftest.py 2009-07-10 07:14:02 +0000
+++ b/bzrlib/tests/blackbox/test_selftest.py 2009-08-23 22:15:51 +0000
@@ -26,6 +26,7 @@
import bzrlib
from bzrlib import (
osutils,
+ tests,
)
from bzrlib.errors import ParamikoNotPresent
from bzrlib.tests import (
@@ -512,59 +513,55 @@
return (header,body,footer)
def test_list_only(self):
- # check that bzr selftest --list-only works correctly
- out,err = self.run_bzr('selftest selftest --list-only')
- (header,body,footer) = self._parse_test_list(out.splitlines())
- num_tests = len(body)
- self.assertLength(0, header)
- self.assertLength(0, footer)
- self.assertEqual('', err)
-
- def test_list_only_filtered(self):
- # check that a filtered --list-only works, both include and exclude
- out_all,err_all = self.run_bzr('selftest --list-only')
- tests_all = self._parse_test_list(out_all.splitlines())[1]
- out_incl,err_incl = self.run_bzr('selftest --list-only selftest')
- tests_incl = self._parse_test_list(out_incl.splitlines())[1]
- self.assertSubset(tests_incl, tests_all)
- out_excl,err_excl = self.run_bzr(['selftest', '--list-only',
- '--exclude', 'selftest'])
- tests_excl = self._parse_test_list(out_excl.splitlines())[1]
- self.assertSubset(tests_excl, tests_all)
- set_incl = set(tests_incl)
- set_excl = set(tests_excl)
- intersection = set_incl.intersection(set_excl)
- self.assertEquals(0, len(intersection))
- self.assertEquals(len(tests_all), len(tests_incl) + len(tests_excl))
-
- def test_list_only_random(self):
- # check that --randomize works correctly
- out_all,err_all = self.run_bzr('selftest --list-only selftest')
- tests_all = self._parse_test_list(out_all.splitlines())[1]
- # XXX: It looks like there are some orders for generating tests that
- # fail as of 20070504 - maybe because of import order dependencies.
- # So unfortunately this will rarely intermittently fail at the moment.
- # -- mbp 20070504
- out_rand,err_rand = self.run_bzr(['selftest', '--list-only',
- 'selftest', '--randomize', 'now'])
- (header_rand,tests_rand,dummy) = self._parse_test_list(
- out_rand.splitlines(), 1)
- # XXX: The following line asserts that the randomized order is not the
- # same as the default order. It is just possible that they'll get
- # randomized into the same order and this will falsely fail, but
- # that's very unlikely in practice because there are thousands of
- # tests.
- self.assertNotEqual(tests_all, tests_rand)
- self.assertEqual(sorted(tests_all), sorted(tests_rand))
- # Check that the seed can be reused to get the exact same order
- seed_re = re.compile('Randomizing test order using seed (\w+)')
- match_obj = seed_re.search(header_rand[-1])
- seed = match_obj.group(1)
- out_rand2,err_rand2 = self.run_bzr(['selftest', '--list-only',
- 'selftest', '--randomize', seed])
- (header_rand2,tests_rand2,dummy) = self._parse_test_list(
- out_rand2.splitlines(), 1)
- self.assertEqual(tests_rand, tests_rand2)
+ # check that bzr selftest --list-only outputs no ui noise
+ def selftest(*args, **kwargs):
+ """Capture the arguments selftest was run with."""
+ return True
+ def outputs_nothing(cmdline):
+ out,err = self.run_bzr(cmdline)
+ (header,body,footer) = self._parse_test_list(out.splitlines())
+ num_tests = len(body)
+ self.assertLength(0, header)
+ self.assertLength(0, footer)
+ self.assertEqual('', err)
+ # Yes this prevents using threads to run the test suite in parallel,
+ # however we don't have a clean dependency injector for commands,
+ # and even if we did - we'd still be testing that the glue is wired
+ # up correctly. XXX: TODO: Solve this testing problem.
+ original_selftest = tests.selftest
+ tests.selftest = selftest
+ try:
+ outputs_nothing('selftest --list-only')
+ outputs_nothing('selftest --list-only selftest')
+ outputs_nothing(['selftest', '--list-only', '--exclude', 'selftest'])
+ finally:
+ tests.selftest = original_selftest
+
+ def test_parameters_passed_to_core(self):
+ params = []
+ def selftest(*args, **kwargs):
+ """Capture the arguments selftest was run with."""
+ params.append((args, kwargs))
+ return True
+ # Yes this prevents using threads to run the test suite in parallel,
+ # however we don't have a clean dependency injector for commands,
+ # and even if we did - we'd still be testing that the glue is wired
+ # up correctly. XXX: TODO: Solve this testing problem.
+ original_selftest = tests.selftest
+ tests.selftest = selftest
+ try:
+ self.run_bzr('selftest --list-only')
+ self.run_bzr('selftest --list-only selftest')
+ self.run_bzr(['selftest', '--list-only', '--exclude', 'selftest'])
+ self.run_bzr(['selftest', '--list-only', 'selftest',
+ '--randomize', 'now'])
+ # list_only should have been passed in each invocation.
+ self.assertTrue("list_only" in params[0][1])
+ self.assertTrue("list_only" in params[1][1])
+ self.assertTrue("list_only" in params[2][1])
+ self.assertSubset(["list_only", "random_seed"], params[2][1])
+ finally:
+ tests.selftest = original_selftest
class TestSelftestWithIdList(TestCaseInTempDir):
=== modified file 'bzrlib/tests/test_selftest.py'
--- a/bzrlib/tests/test_selftest.py 2009-08-17 03:47:03 +0000
+++ b/bzrlib/tests/test_selftest.py 2009-08-23 22:15:51 +0000
@@ -1770,6 +1770,60 @@
test_suite_factory=factory)
self.assertEqual([True], factory_called)
+ def factory(self):
+ """A test suite factory."""
+ class Test(tests.TestCase):
+ def a(self):
+ pass
+ def b(self):
+ pass
+ def c(self):
+ pass
+ return TestUtil.TestSuite([Test("a"), Test("b"), Test("c")])
+
+ def run_selftest(self, **kwargs):
+ """Run selftest returning its output."""
+ output = StringIO()
+ self.assertEqual(True, tests.selftest(stream=output, **kwargs))
+ output.seek(0)
+ return output
+
+ def test_list_only(self):
+ output = self.run_selftest(test_suite_factory=self.factory,
+ list_only=True)
+ self.assertEqual(3, len(output.readlines()))
+
+ def test_list_only_filtered(self):
+ output = self.run_selftest(test_suite_factory=self.factory,
+ list_only=True, pattern="Test.b")
+ self.assertEndsWith(output.getvalue(), "Test.b\n")
+ self.assertLength(1, output.readlines())
+
+ def test_list_only_excludes(self):
+ output = self.run_selftest(test_suite_factory=self.factory,
+ list_only=True, exclude_pattern="Test.b")
+ self.assertNotContainsRe("Test.b", output.getvalue())
+ self.assertLength(2, output.readlines())
+
+ def test_random(self):
+ # test randomising by listing a number of tests.
+ output_123 = self.run_selftest(test_suite_factory=self.factory,
+ list_only=True, random_seed="123")
+ output_234 = self.run_selftest(test_suite_factory=self.factory,
+ list_only=True, random_seed="234")
+ self.assertNotEqual(output_123, output_234)
+ # "Randominzing test order..\n\n
+ self.assertLength(5, output_123.readlines())
+ self.assertLength(5, output_234.readlines())
+
+ def test_random_reuse_is_same_order(self):
+ # test randomising by listing a number of tests.
+ expected = self.run_selftest(test_suite_factory=self.factory,
+ list_only=True, random_seed="123")
+ repeated = self.run_selftest(test_suite_factory=self.factory,
+ list_only=True, random_seed="123")
+ self.assertEqual(expected.getvalue(), repeated.getvalue())
+
class TestKnownFailure(tests.TestCase):
More information about the bazaar-commits
mailing list