[apparmor] [patch 04/13] parser - add simple valgrind wrapper tests
Seth Arnold
seth.arnold at canonical.com
Thu Nov 28 07:57:30 UTC 2013
On Thu, Oct 10, 2013 at 01:46:18PM -0700, Steve Beattie wrote:
> This patch adds a test wrapper that runs valgrind on the parser over the
> simple_tests tree (or other directory tree if passed on the command
> line). An alternate parser location can also be passed on the command
> line.
>
> Like the libapparmor python bindings test, this test uses a bit of magic
> to generate tests that doesn't work with auto-detecting test utilities
> like nose.
>
> Running valgrind on the parser over all 69000+ testcases takes several
> hours, so while this patch includes a make target 'make valgrind', it
> does not add it to the set of tests run when 'make check' is called.
> Perhaps a 'make extra-tests' target is in order.
>
> Patch history:
> v1: - initial version.
> v2: - add some valgrind suppressions for overaggressive 4 byte reads
> past the end of allocated storage (not completed).
> v3: - add ability to dump valgrind suppressions to stdout, to use
> diagnosis runs of valgrind for determining whether a given
> failure is a false positive or not.
> - correctly return 0 on a successful run and an error code if one
> or more test cases fail.
> - point LD_LIBRARY_PATH at the in-tree libapparmor build.
> - split out some utility functions into testlib.py, for possible
> use by other to be written test scripts
>
> Signed-off-by: Steve Beattie <steve at nxnw.org>
> Acked-by: Tyler Hicks <tyhicks at canonical.com> (for v2 version)
This is cool. :)
Acked-by: Seth Arnold <seth.arnold at canonical.com>
Note that the 'valgrind' make target should be added to the .PHONY: list
of fake targets.
Thanks!
>
> ---
> parser/tst/Makefile | 3
> parser/tst/testlib.py | 79 ++++++++++++++++++++++
> parser/tst/valgrind_simple.py | 147 ++++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 229 insertions(+)
>
> Index: b/parser/tst/valgrind_simple.py
> ===================================================================
> --- /dev/null
> +++ b/parser/tst/valgrind_simple.py
> @@ -0,0 +1,147 @@
> +#!/usr/bin/env python
> +# ------------------------------------------------------------------
> +#
> +# Copyright (C) 2013 Canonical Ltd.
> +# Author: Steve Beattie <steve at nxnw.org>
> +#
> +# This program is free software; you can redistribute it and/or
> +# modify it under the terms of version 2 of the GNU General Public
> +# License published by the Free Software Foundation.
> +#
> +# ------------------------------------------------------------------
> +
> +# TODO
> +# - finish adding suppressions for valgrind false positives
> +
> +from optparse import OptionParser # deprecated, should move to argparse eventually
> +import os
> +import tempfile
> +import unittest
> +import testlib
> +
> +DEFAULT_TESTDIR = "./simple_tests/vars"
> +VALGRIND_ERROR_CODE = 151
> +VALGRIND_ARGS = ['--leak-check=full', '--error-exitcode=%d' % (VALGRIND_ERROR_CODE)]
> +
> +VALGRIND_SUPPRESSIONS = '''
> +{
> + valgrind-add_search_dir-obsessive-overreads
> + Memcheck:Addr4
> + fun:_Z*add_search_dir*
> + fun:_Z*process_arg*
> + fun:main
> +}
> +
> +{
> + valgrind-yylex-obsessive-overreads
> + Memcheck:Addr4
> + fun:_Z?yylex?
> + fun:_Z*yyparse*
> + fun:_Z*process_profile*
> + fun:main
> +}
> +
> +{
> + valgrind-serialize_profile-obsessive-overreads
> + Memcheck:Addr4
> + fun:_Z*sd_serialize_profile*
> + fun:_Z*sd_serialize_codomain*
> + fun:_Z*load_codomain*
> + fun:_Z*__load_flattened_hat*
> + ...
> + fun:twalk
> + fun:_Z*load_flattened_hats*
> + fun:_Z*sd_serialize_codomain*
> + fun:_Z*load_codomain*
> + fun:_Z*__load_policy*
> +}'''
> +
> +
> +class AAParserValgrindTests(unittest.TestCase):
> + def setUp(self):
> + # REPORT ALL THE OUTPUT
> + self.maxDiff = None
> +
> + def _runtest(self, testname, config):
> + parser_args = ['-Q', '-I', config.testdir]
> + failure_rc = [VALGRIND_ERROR_CODE, testlib.TIMEOUT_ERROR_CODE]
> + command = ['valgrind']
> + command.extend(VALGRIND_ARGS)
> + command.append(config.parser)
> + command.extend(parser_args)
> + command.append(testname)
> + rc, output = testlib.run_cmd(command, timeout=120)
> + self.assertNotIn(rc, failure_rc,
> + "valgrind returned error code %d, gave the following output\n%s" % (rc, output))
> +
> +
> +def find_testcases(testdir):
> + '''dig testcases out of passed directory'''
> +
> + for (fdir, direntries, files) in os.walk(testdir):
> + for f in files:
> + if f.endswith(".sd"):
> + yield os.path.join(fdir, f)
> +
> +
> +def create_suppressions():
> + '''generate valgrind suppressions file'''
> +
> + handle, name = tempfile.mkstemp(suffix='.suppressions', prefix='aa-parser-valgrind')
> + os.close(handle)
> + handle = open(name,"w+")
> + handle.write(VALGRIND_SUPPRESSIONS)
> + handle.close()
> + return name
> +
> +def main():
> + usage = "usage: %prog [options] [test_directory]"
> + p = OptionParser(usage=usage)
> + p.add_option('-p', '--parser', default=testlib.DEFAULT_PARSER, action="store", type="string", dest='parser')
> + p.add_option('-v', '--verbose', action="store_true", dest="verbose")
> + p.add_option('-s', '--skip-suppressions', action="store_true", dest="skip_suppressions",
> + help="Don't use valgrind suppressions to skip false positives")
> + p.add_option('--dump-suppressions', action="store_true", dest="dump_suppressions",
> + help="Dump valgrind suppressions to stdout")
> + config, args = p.parse_args()
> +
> + if config.dump_suppressions:
> + print(VALGRIND_SUPPRESSIONS)
> + return rc
> +
> + verbosity = 1
> + if config.verbose:
> + verbosity = 2
> +
> + if len(args) == 1:
> + config.testdir = args[0]
> + else:
> + config.testdir = DEFAULT_TESTDIR
> +
> + if not config.skip_suppressions:
> + suppression_file = create_suppressions()
> + VALGRIND_ARGS.append('--suppressions=%s' % (suppression_file))
> +
> + for f in find_testcases(config.testdir):
> + def stub_test(self, testname=f):
> + self._runtest(testname, config)
> + stub_test.__doc__ = "test %s" % (f)
> + setattr(AAParserValgrindTests, 'test_%s' % (f), stub_test)
> + test_suite = unittest.TestSuite()
> + test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(AAParserValgrindTests))
> +
> + rc = 0
> + try:
> + result = unittest.TextTestRunner(verbosity=verbosity).run(test_suite)
> + if not result.wasSuccessful():
> + rc = 1
> + except:
> + rc = 1
> + finally:
> + os.remove(suppression_file)
> +
> + return rc
> +
> +if __name__ == "__main__":
> + rc = main()
> + sys.exit(rc)
> Index: b/parser/tst/Makefile
> ===================================================================
> --- a/parser/tst/Makefile
> +++ b/parser/tst/Makefile
> @@ -51,6 +51,9 @@ minimize: $(PARSER)
> equality: $(PARSER)
> LANG=C LD_LIBRARY_PATH=$(LIBAPPARMOR_LDPATH) APPARMOR_PARSER="$(PARSER)" ./equality.sh
>
> +valgrind: $(PARSER)
> + LANG=C LD_LIBRARY_PATH=$(LIBAPPARMOR_LDPATH) ./valgrind_simple.py -p "$(PARSER)" -v simple_tests
> +
> $(PARSER):
> make -C $(PARSER_DIR) $(PARSER_BIN)
>
> Index: b/parser/tst/testlib.py
> ===================================================================
> --- /dev/null
> +++ b/parser/tst/testlib.py
> @@ -0,0 +1,79 @@
> +#!/usr/bin/env python
> +# ------------------------------------------------------------------
> +#
> +# Copyright (C) 2013 Canonical Ltd.
> +# Author: Steve Beattie <steve at nxnw.org>
> +#
> +# This program is free software; you can redistribute it and/or
> +# modify it under the terms of version 2 of the GNU General Public
> +# License published by the Free Software Foundation.
> +#
> +# ------------------------------------------------------------------
> +
> +import signal
> +import subprocess
> +
> +TIMEOUT_ERROR_CODE = 152
> +DEFAULT_PARSER = '../apparmor_parser'
> +
> +
> +# http://www.chiark.greenend.org.uk/ucgi/~cjwatson/blosxom/2009-07-02-python-sigpipe.html
> +# This is needed so that the subprocesses that produce endless output
> +# actually quit when the reader goes away.
> +def subprocess_setup():
> + # Python installs a SIGPIPE handler by default. This is usually not
> + # what non-Python subprocesses expect.
> + signal.signal(signal.SIGPIPE, signal.SIG_DFL)
> +
> +
> +def run_cmd(command, input=None, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, stdin=None, timeout=120):
> + '''Try to execute given command (array) and return its stdout, or
> + return a textual error if it failed.'''
> +
> + try:
> + sp = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr, close_fds=True, preexec_fn=subprocess_setup)
> + except OSError as e:
> + return [127, str(e)]
> +
> + timeout_communicate = TimeoutFunction(sp.communicate, timeout)
> + out, outerr = (None, None)
> + try:
> + out, outerr = timeout_communicate(input)
> + rc = sp.returncode
> + except TimeoutFunctionException as e:
> + sp.terminate()
> + outerr = b'test timed out, killed'
> + rc = TIMEOUT_ERROR_CODE
> +
> + # Handle redirection of stdout
> + if out is None:
> + out = b''
> + # Handle redirection of stderr
> + if outerr is None:
> + outerr = b''
> + return [rc, out.decode('utf-8') + outerr.decode('utf-8')]
> +
> +
> +# Timeout handler using alarm() from John P. Speno's Pythonic Avocado
> +class TimeoutFunctionException(Exception):
> + """Exception to raise on a timeout"""
> + pass
> +
> +
> +class TimeoutFunction:
> + def __init__(self, function, timeout):
> + self.timeout = timeout
> + self.function = function
> +
> + def handle_timeout(self, signum, frame):
> + raise TimeoutFunctionException()
> +
> + def __call__(self, *args, **kwargs):
> + old = signal.signal(signal.SIGALRM, self.handle_timeout)
> + signal.alarm(self.timeout)
> + try:
> + result = self.function(*args, **kwargs)
> + finally:
> + signal.signal(signal.SIGALRM, old)
> + signal.alarm(0)
> + return result
>
>
> --
> AppArmor mailing list
> AppArmor at lists.ubuntu.com
> Modify settings or unsubscribe at: https://lists.ubuntu.com/mailman/listinfo/apparmor
>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 490 bytes
Desc: Digital signature
URL: <https://lists.ubuntu.com/archives/apparmor/attachments/20131127/164af963/attachment-0001.pgp>
More information about the AppArmor
mailing list