# Bazaar revision bundle v0.8 # # message: # (lifeless, cfbolz): Add recording of benchmark results to the benchmark history # file ".perf_history". # # committer: Carl Friedrich Bolz # date: Wed 2006-07-05 14:27:59.631119967 +0200 === modified file bzrlib/builtins.py --- bzrlib/builtins.py +++ bzrlib/builtins.py @@ -1890,17 +1890,24 @@ test_suite_factory = benchmarks.test_suite if verbose is None: verbose = True + benchfile = open(".perf_history", "at") else: test_suite_factory = None if verbose is None: verbose = False - result = selftest(verbose=verbose, - pattern=pattern, - stop_on_failure=one, - keep_output=keep_output, - transport=transport, - test_suite_factory=test_suite_factory, - lsprof_timed=lsprof_timed) + benchfile = None + try: + result = selftest(verbose=verbose, + pattern=pattern, + stop_on_failure=one, + keep_output=keep_output, + transport=transport, + test_suite_factory=test_suite_factory, + lsprof_timed=lsprof_timed, + bench_history=benchfile) + finally: + if benchfile is not None: + benchfile.close() if result: info('tests passed') else: === modified file bzrlib/tests/__init__.py --- bzrlib/tests/__init__.py +++ bzrlib/tests/__init__.py @@ -132,9 +132,13 @@ """ stop_early = False - def __init__(self, stream, descriptions, verbosity, pb=None): + def __init__(self, stream, descriptions, verbosity, pb=None, + bench_history=None): unittest._TextTestResult.__init__(self, stream, descriptions, verbosity) self.pb = pb + if bench_history is not None: + bench_history.write("--date %s\n" % time.time()) + self._bench_history = bench_history def extractBenchmarkTime(self, testCase): """Add a benchmark time for the current test case.""" @@ -238,6 +242,11 @@ def addSuccess(self, test): self.extractBenchmarkTime(test) + if self._bench_history is not None: + if self._benchmarkTime is not None: + self._bench_history.write("%s %s\n" % ( + self._formatTime(self._benchmarkTime), + test.id())) if self.showAll: self.stream.writeln(' OK %s' % self._testTimeString()) for bench_called, stats in getattr(test, '_benchcalls', []): @@ -287,18 +296,21 @@ descriptions=0, verbosity=1, keep_output=False, - pb=None): + pb=None, + bench_history=None): self.stream = unittest._WritelnDecorator(stream) self.descriptions = descriptions self.verbosity = verbosity self.keep_output = keep_output self.pb = pb + self._bench_history = bench_history def _makeResult(self): result = _MyResult(self.stream, self.descriptions, self.verbosity, - pb=self.pb) + pb=self.pb, + bench_history=self._bench_history) result.stop_early = self.stop_on_failure return result @@ -1151,7 +1163,7 @@ def run_suite(suite, name='test', verbose=False, pattern=".*", stop_on_failure=False, keep_output=False, - transport=None, lsprof_timed=None): + transport=None, lsprof_timed=None, bench_history=None): TestCaseInTempDir._TEST_NAME = name TestCase._gather_lsprof_in_benchmarks = lsprof_timed if verbose: @@ -1164,7 +1176,8 @@ descriptions=0, verbosity=verbosity, keep_output=keep_output, - pb=pb) + pb=pb, + bench_history=bench_history) runner.stop_on_failure=stop_on_failure if pattern != '.*': suite = filter_suite_by_re(suite, pattern) @@ -1176,7 +1189,8 @@ keep_output=False, transport=None, test_suite_factory=None, - lsprof_timed=None): + lsprof_timed=None, + bench_history=None): """Run the whole test suite under the enhanced runner""" global default_transport if transport is None: @@ -1191,7 +1205,8 @@ return run_suite(suite, 'testbzr', verbose=verbose, pattern=pattern, stop_on_failure=stop_on_failure, keep_output=keep_output, transport=transport, - lsprof_timed=lsprof_timed) + lsprof_timed=lsprof_timed, + bench_history=bench_history) finally: default_transport = old_transport === modified file bzrlib/tests/blackbox/test_selftest.py --- bzrlib/tests/blackbox/test_selftest.py +++ bzrlib/tests/blackbox/test_selftest.py @@ -22,6 +22,7 @@ from bzrlib.tests import ( TestCase, TestCaseInTempDir, + TestCaseWithTransport, TestSkipped, ) from bzrlib.tests.blackbox import ExternalBase @@ -77,16 +78,6 @@ TestOptions.current_test = None TestCaseInTempDir.TEST_ROOT = old_root - def test_benchmark_runs_benchmark_tests(self): - """bzr selftest --benchmark should not run the default test suite.""" - # We test this by passing a regression test name to --benchmark, which - # should result in 0 rests run. - out, err = self.run_bzr('selftest', '--benchmark', 'workingtree_implementations') - self.assertContainsRe(out, 'Ran 0 tests.*\n\nOK') - self.assertEqual( - 'running tests...\ntests passed\n', - err) - class TestRunBzr(ExternalBase): @@ -105,6 +96,31 @@ self.assertEqual('zippy', self.stdin) +class TestBenchmarkTests(TestCaseWithTransport): + + def test_benchmark_runs_benchmark_tests(self): + """bzr selftest --benchmark should not run the default test suite.""" + # We test this by passing a regression test name to --benchmark, which + # should result in 0 rests run. + old_root = TestCaseInTempDir.TEST_ROOT + try: + TestCaseInTempDir.TEST_ROOT = None + out, err = self.run_bzr('selftest', '--benchmark', 'workingtree_implementations') + finally: + TestCaseInTempDir.TEST_ROOT = old_root + self.assertContainsRe(out, 'Ran 0 tests.*\n\nOK') + self.assertEqual( + 'running tests...\ntests passed\n', + err) + benchfile = open(".perf_history", "rt") + try: + lines = benchfile.readlines() + finally: + benchfile.close() + self.assertEqual(1, len(lines)) + self.assertContainsRe(lines[0], "--date [0-9.]+") + + class TestRunBzrCaptured(ExternalBase): def apply_redirected(self, stdin=None, stdout=None, stderr=None, === modified file bzrlib/tests/test_selftest.py --- bzrlib/tests/test_selftest.py +++ bzrlib/tests/test_selftest.py @@ -469,6 +469,36 @@ # cheat. Yes, wash thy mouth out with soap. self._benchtime = None + def test_assigned_benchmark_file_stores_date(self): + output = StringIO() + result = bzrlib.tests._MyResult(self._log_file, + descriptions=0, + verbosity=1, + bench_history=output + ) + output_string = output.getvalue() + self.assertContainsRe(output_string, "--date [0-9.]+\n") + + def test_benchhistory_records_test_times(self): + result_stream = StringIO() + result = bzrlib.tests._MyResult( + self._log_file, + descriptions=0, + verbosity=1, + bench_history=result_stream + ) + + # we want profile a call and check that its test duration is recorded + # make a new test instance that when run will generate a benchmark + example_test_case = TestTestResult("_time_hello_world_encoding") + # execute the test, which should succeed and record times + example_test_case.run(result) + lines = result_stream.getvalue().splitlines() + self.assertEqual(2, len(lines)) + self.assertContainsRe(lines[1], + " *[0-9]+ms bzrlib.tests.test_selftest.TestTestResult" + "._time_hello_world_encoding") + def _time_hello_world_encoding(self): """Profile two sleep calls @@ -563,6 +593,14 @@ result = self.run_test_runner(runner, test) self.assertTrue(result.wasSuccessful()) + def test_bench_history(self): + test = TestRunner('dummy_test') + output = StringIO() + runner = TextTestRunner(stream=self._log_file, bench_history=output) + result = self.run_test_runner(runner, test) + output_string = output.getvalue() + self.assertContainsRe(output_string, "--date [0-9.]+\n") + class TestTestCase(TestCase): """Tests that test the core bzrlib TestCase.""" # revision id: cfbolz@gmx.de-20060705122759-0a3481a4647b16dc # sha1: 4d9220def8aab20a29459c667883b11df45d06b4 # inventory sha1: 7a0fb6c0a17e97d80b721d20ba316eacb3aa4a78 # parent ids: # cfbolz@gmx.de-20060705113854-03d5196875cbe3c2 # base id: pqm@pqm.ubuntu.com-20060628140650-135f9ed05d63075c # properties: # branch-nick: bzr.performance-history # message: # (lifeless, cfbolz, hpk): Add a benchmark output parameter to TextTestRunner. # # committer: Carl Friedrich Bolz # date: Wed 2006-07-05 13:38:54.961703062 +0200 === modified file bzrlib/tests/__init__.py // encoding:base64 LS0tIGJ6cmxpYi90ZXN0cy9fX2luaXRfXy5weQorKysgYnpybGliL3Rlc3RzL19faW5pdF9fLnB5 CkBAIC0yOTAsMTggKzI5MCwyMSBAQAogICAgICAgICAgICAgICAgICBkZXNjcmlwdGlvbnM9MCwK ICAgICAgICAgICAgICAgICAgdmVyYm9zaXR5PTEsCiAgICAgICAgICAgICAgICAgIGtlZXBfb3V0 cHV0PUZhbHNlLAotICAgICAgICAgICAgICAgICBwYj1Ob25lKToKKyAgICAgICAgICAgICAgICAg cGI9Tm9uZSwKKyAgICAgICAgICAgICAgICAgYmVuY2hfaGlzdG9yeT1Ob25lKToKICAgICAgICAg c2VsZi5zdHJlYW0gPSB1bml0dGVzdC5fV3JpdGVsbkRlY29yYXRvcihzdHJlYW0pCiAgICAgICAg IHNlbGYuZGVzY3JpcHRpb25zID0gZGVzY3JpcHRpb25zCiAgICAgICAgIHNlbGYudmVyYm9zaXR5 ID0gdmVyYm9zaXR5CiAgICAgICAgIHNlbGYua2VlcF9vdXRwdXQgPSBrZWVwX291dHB1dAogICAg ICAgICBzZWxmLnBiID0gcGIKKyAgICAgICAgc2VsZi5fYmVuY2hfaGlzdG9yeSA9IGJlbmNoX2hp c3RvcnkKIAogICAgIGRlZiBfbWFrZVJlc3VsdChzZWxmKToKICAgICAgICAgcmVzdWx0ID0gX015 UmVzdWx0KHNlbGYuc3RyZWFtLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgc2VsZi5kZXNj cmlwdGlvbnMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBzZWxmLnZlcmJvc2l0eSwKLSAg ICAgICAgICAgICAgICAgICAgICAgICAgIHBiPXNlbGYucGIpCisgICAgICAgICAgICAgICAgICAg ICAgICAgICBwYj1zZWxmLnBiLAorICAgICAgICAgICAgICAgICAgICAgICAgICAgYmVuY2hfaGlz dG9yeT1zZWxmLl9iZW5jaF9oaXN0b3J5KQogICAgICAgICByZXN1bHQuc3RvcF9lYXJseSA9IHNl bGYuc3RvcF9vbl9mYWlsdXJlCiAgICAgICAgIHJldHVybiByZXN1bHQKIAoK === modified file bzrlib/tests/test_selftest.py // encoding:base64 LS0tIGJ6cmxpYi90ZXN0cy90ZXN0X3NlbGZ0ZXN0LnB5CisrKyBienJsaWIvdGVzdHMvdGVzdF9z ZWxmdGVzdC5weQpAQCAtNDc5LDcgKzQ3OSw2IEBACiAgICAgICAgIG91dHB1dF9zdHJpbmcgPSBv dXRwdXQuZ2V0dmFsdWUoKQogICAgICAgICBzZWxmLmFzc2VydENvbnRhaW5zUmUob3V0cHV0X3N0 cmluZywgIi0tZGF0ZSBbMC05Ll0rIikKIAotCiAgICAgZGVmIF90aW1lX2hlbGxvX3dvcmxkX2Vu Y29kaW5nKHNlbGYpOgogICAgICAgICAiIiJQcm9maWxlIHR3byBzbGVlcCBjYWxscwogICAgICAg ICAKQEAgLTU3NCw2ICs1NzMsMTQgQEAKICAgICAgICAgcmVzdWx0ID0gc2VsZi5ydW5fdGVzdF9y dW5uZXIocnVubmVyLCB0ZXN0KQogICAgICAgICBzZWxmLmFzc2VydFRydWUocmVzdWx0Lndhc1N1 Y2Nlc3NmdWwoKSkKIAorICAgIGRlZiB0ZXN0X2JlbmNoX2hpc3Rvcnkoc2VsZik6CisgICAgICAg IHRlc3QgPSBUZXN0UnVubmVyKCdkdW1teV90ZXN0JykKKyAgICAgICAgb3V0cHV0ID0gU3RyaW5n SU8oKQorICAgICAgICBydW5uZXIgPSBUZXh0VGVzdFJ1bm5lcihzdHJlYW09c2VsZi5fbG9nX2Zp bGUsIGJlbmNoX2hpc3Rvcnk9b3V0cHV0KQorICAgICAgICByZXN1bHQgPSBzZWxmLnJ1bl90ZXN0 X3J1bm5lcihydW5uZXIsIHRlc3QpCisgICAgICAgIG91dHB1dF9zdHJpbmcgPSBvdXRwdXQuZ2V0 dmFsdWUoKQorICAgICAgICBzZWxmLmFzc2VydENvbnRhaW5zUmUob3V0cHV0X3N0cmluZywgIi0t ZGF0ZSBbMC05Ll0rIikKKwogCiBjbGFzcyBUZXN0VGVzdENhc2UoVGVzdENhc2UpOgogICAgICIi IlRlc3RzIHRoYXQgdGVzdCB0aGUgY29yZSBienJsaWIgVGVzdENhc2UuIiIiCgo= # revision id: cfbolz@gmx.de-20060705113854-03d5196875cbe3c2 # sha1: 14f684276d60b9b878da4423b5bad2281e447899 # inventory sha1: b8b3ed40a0f858398b06b0f4eb32571fffbe6c37 # parent ids: # cfbolz@gmx.de-20060705113240-5fd89b1f29551806 # properties: # branch-nick: bzr.performance-history # message: # (lifeless, cfbolz, hpk): Give the test result object an optional benchmark # output file. # # committer: Carl Friedrich Bolz # date: Wed 2006-07-05 13:32:40.314552069 +0200 === modified file bzrlib/tests/__init__.py // encoding:base64 LS0tIGJ6cmxpYi90ZXN0cy9fX2luaXRfXy5weQorKysgYnpybGliL3Rlc3RzL19faW5pdF9fLnB5 CkBAIC0xMzIsOSArMTMyLDEyIEBACiAgICAgIiIiCiAgICAgc3RvcF9lYXJseSA9IEZhbHNlCiAg ICAgCi0gICAgZGVmIF9faW5pdF9fKHNlbGYsIHN0cmVhbSwgZGVzY3JpcHRpb25zLCB2ZXJib3Np dHksIHBiPU5vbmUpOgorICAgIGRlZiBfX2luaXRfXyhzZWxmLCBzdHJlYW0sIGRlc2NyaXB0aW9u cywgdmVyYm9zaXR5LCBwYj1Ob25lLAorICAgICAgICAgICAgICAgICBiZW5jaF9oaXN0b3J5PU5v bmUpOgogICAgICAgICB1bml0dGVzdC5fVGV4dFRlc3RSZXN1bHQuX19pbml0X18oc2VsZiwgc3Ry ZWFtLCBkZXNjcmlwdGlvbnMsIHZlcmJvc2l0eSkKICAgICAgICAgc2VsZi5wYiA9IHBiCisgICAg ICAgIGlmIGJlbmNoX2hpc3RvcnkgaXMgbm90IE5vbmU6CisgICAgICAgICAgICBiZW5jaF9oaXN0 b3J5LndyaXRlKCItLWRhdGUgJXMiICUgdGltZS50aW1lKCkpCiAgICAgCiAgICAgZGVmIGV4dHJh Y3RCZW5jaG1hcmtUaW1lKHNlbGYsIHRlc3RDYXNlKToKICAgICAgICAgIiIiQWRkIGEgYmVuY2ht YXJrIHRpbWUgZm9yIHRoZSBjdXJyZW50IHRlc3QgY2FzZS4iIiIKCg== === modified file bzrlib/tests/test_selftest.py // encoding:base64 LS0tIGJ6cmxpYi90ZXN0cy90ZXN0X3NlbGZ0ZXN0LnB5CisrKyBienJsaWIvdGVzdHMvdGVzdF9z ZWxmdGVzdC5weQpAQCAtNDY5LDYgKzQ2OSwxNyBAQAogICAgICAgICAjIGNoZWF0LiBZZXMsIHdh c2ggdGh5IG1vdXRoIG91dCB3aXRoIHNvYXAuCiAgICAgICAgIHNlbGYuX2JlbmNodGltZSA9IE5v bmUKIAorICAgIGRlZiB0ZXN0X2Fzc2lnbmVkX2JlbmNobWFya19maWxlX3N0b3Jlc19kYXRlKHNl bGYpOgorICAgICAgICBvdXRwdXQgPSBTdHJpbmdJTygpCisgICAgICAgIHJlc3VsdCA9IGJ6cmxp Yi50ZXN0cy5fTXlSZXN1bHQoc2VsZi5fbG9nX2ZpbGUsCisgICAgICAgICAgICAgICAgICAgICAg ICAgICAgICAgICAgICAgICAgZGVzY3JpcHRpb25zPTAsCisgICAgICAgICAgICAgICAgICAgICAg ICAgICAgICAgICAgICAgICAgdmVyYm9zaXR5PTEsCisgICAgICAgICAgICAgICAgICAgICAgICAg ICAgICAgICAgICAgICAgYmVuY2hfaGlzdG9yeT1vdXRwdXQKKyAgICAgICAgICAgICAgICAgICAg ICAgICAgICAgICAgICAgICAgICApCisgICAgICAgIG91dHB1dF9zdHJpbmcgPSBvdXRwdXQuZ2V0 dmFsdWUoKQorICAgICAgICBzZWxmLmFzc2VydENvbnRhaW5zUmUob3V0cHV0X3N0cmluZywgIi0t ZGF0ZSBbMC05Ll0rIikKKworCiAgICAgZGVmIF90aW1lX2hlbGxvX3dvcmxkX2VuY29kaW5nKHNl bGYpOgogICAgICAgICAiIiJQcm9maWxlIHR3byBzbGVlcCBjYWxscwogICAgICAgICAKCg== # revision id: cfbolz@gmx.de-20060705113240-5fd89b1f29551806 # sha1: db8841b229e641fd4ab391bea580774a936872f9 # inventory sha1: d742f2871f3eba90a9800d3bcc5ee1d8cec24b4b # parent ids: # pqm@pqm.ubuntu.com-20060628140650-135f9ed05d63075c # properties: # branch-nick: bzr.performance-history