ACK: [ACT][PATCH] UBUNTU: SAUCE: ubuntu_kernel_selftests: granularity improvement
Sean Feole
sean.feole at canonical.com
Tue Nov 9 13:55:28 UTC 2021
+1 from me
Thanks for putting in the time to finally make this right. It was not
an easy task, if anything more of a headache.
I know you have been testing this around the clock thanks for your hard
work on this!
-Sfeole
On Tue, 9 Nov 2021 16:32:55 +0800
Po-Hsu Lin <po-hsu.lin at canonical.com> wrote:
> BugLink: https://bugs.launchpad.net/bugs/1941052
>
> This patch will change how we run those test cases in the selftests
> directory of a kernel tree:
> * Test will now abort early if the setup() has failed
> * Each test suite will be built first with a SUITE_NAME-build job,
> we will be able to distinguish build failures and actual test
> failures more easily.
> * Each test inside a suite will be executed one-by-one with name in
> a format of SUITE:CASE, e.g. net:test_bpf.sh, this will help us
> to improve our result hinting process, make the test report
> become more reliable and easier to gate new regressions.
> * Use a helper.mk file to get the executable test names, by doing
> so we can workaround the issue caused by kernel source code
> difference across various releases.
> * Remove the summary section, we won't have a summary for a whole
> suite in the end of the test. I think this feature can be added
> back if needed.
> * Remove out-dated control files.
> * Trusty control file is untouched as this test is blacklisted.
>
> This patch has been tested on KVM instances with X/F/H/I kernel on our
> dev jenkins and X-4.15/B-4.15/B-5.4/F/H/I Oracle kernel with SRU cycle
> sru-20211018, the overall result is looking good. Bionic kernel will
> need fix in bug 1949889 to land first to make net tests working
> properly.
>
> Signed-off-by: Po-Hsu Lin <po-hsu.lin at canonical.com>
> ---
> ubuntu_kernel_selftests/control | 28 ++++--
> ubuntu_kernel_selftests/control.ubuntu.artful | 21 -----
> ubuntu_kernel_selftests/control.ubuntu.utopic | 21 -----
> ubuntu_kernel_selftests/control.ubuntu.vivid | 21 -----
> ubuntu_kernel_selftests/control.ubuntu.wily | 21 -----
> ubuntu_kernel_selftests/control.ubuntu.xenial | 24 +++--
> .../control.ubuntu.yakkety | 21 -----
> ubuntu_kernel_selftests/control.ubuntu.zesty | 21 -----
> ubuntu_kernel_selftests/helper.mk | 2 +
> .../ubuntu_kernel_selftests.py | 88
> +++++++++---------- 10 files changed, 84 insertions(+), 184
> deletions(-) delete mode 100644
> ubuntu_kernel_selftests/control.ubuntu.artful delete mode 100644
> ubuntu_kernel_selftests/control.ubuntu.utopic delete mode 100644
> ubuntu_kernel_selftests/control.ubuntu.vivid delete mode 100644
> ubuntu_kernel_selftests/control.ubuntu.wily delete mode 100644
> ubuntu_kernel_selftests/control.ubuntu.yakkety delete mode 100644
> ubuntu_kernel_selftests/control.ubuntu.zesty create mode 100755
> ubuntu_kernel_selftests/helper.mk
>
> diff --git a/ubuntu_kernel_selftests/control
> b/ubuntu_kernel_selftests/control index e2874196..4fcee658 100644
> --- a/ubuntu_kernel_selftests/control
> +++ b/ubuntu_kernel_selftests/control
> @@ -1,5 +1,5 @@
> AUTHOR = "Ubuntu"
> -NAME = "selftests"
> +NAME = 'ubuntu_kernel_selftests'
> CRITERIA = """
> Uses built-in kernel repository self tests.
> """
> @@ -10,20 +10,32 @@ TEST_CATEGORY = 'Functional'
> TEST_TYPE = "client"
> DOC = ""
>
> -name = 'ubuntu_kernel_selftests'
> -
> -tests = [
> 'setup','breakpoints','cpu-hotplug','efivarfs','memfd','memory-hotplug','mount','net','ptrace','seccomp','timers','powerpc','user','ftrace'
> ] +categories = ['breakpoints', 'cpu-hotplug', 'efivarfs', 'memfd',
> 'memory-hotplug', 'mount', 'net', 'ptrace', 'seccomp', 'timers',
> 'powerpc', 'user', 'ftrace'] # # The seccomp tests on 4.19+ on
> non-x86 are known to be fail and # need fixing up. For now, disable
> them. #
> release = platform.release().split(".")[:2]
> release = int(release[0])*100 + int(release[1])
> -if 'seccomp' in tests and release > 418 and platform.machine() not
> in ['x86_64', 'ppc64le']:
> - tests.remove('seccomp')
> +if 'seccomp' in categories and release > 418 and platform.machine()
> not in ['x86_64', 'ppc64le']:
> + categories.remove('seccomp')
>
> -for test in tests:
> - results = job.run_test_detail('ubuntu_kernel_selftests',
> test_name=test, tag=test, timeout=60*30) +result =
> job.run_test_detail(NAME, test_name='setup', tag='setup',
> timeout=60*30) +if result == 'ERROR':
> + print("ERROR: test failed to build, skipping all the sub tests")
> +else:
> + for category in categories:
> + build = '{}-build'.format(category)
> + job.run_test_detail(NAME, test_name=build, tag=build,
> timeout=60*10)
> + mk_helper = os.path.join(job.testdir, NAME, 'helper.mk')
> + dir_src = os.path.join(job.bindir, 'tmp', NAME, 'src',
> 'linux/tools/testing/selftests/', category)
> + mk_src = os.path.join(dir_src, 'Makefile')
> + os.chdir(dir_src)
> + cmd = 'make -f {} -f {} gettests'.format(mk_helper, mk_src)
> + tests = utils.system_output(cmd).split()
> + for item in tests:
> + test = "{}:{}".format(category, os.path.basename(item))
> + job.run_test_detail(NAME, test_name=test, tag=test,
> timeout=60*20)
> # vi:set ts=4 sw=4 expandtab syntax=python:
> diff --git a/ubuntu_kernel_selftests/control.ubuntu.artful
> b/ubuntu_kernel_selftests/control.ubuntu.artful deleted file mode
> 100644 index ec5121c9..00000000
> --- a/ubuntu_kernel_selftests/control.ubuntu.artful
> +++ /dev/null
> @@ -1,21 +0,0 @@
> -AUTHOR = "Ubuntu"
> -NAME = "selftests"
> -CRITERIA = """
> -Uses built-in kernel repository self tests.
> -"""
> -SUITE = "None"
> -TIME = "SHORT"
> -TEST_CLASS = 'kernel'
> -TEST_CATEGORY = 'Functional'
> -TEST_TYPE = "client"
> -DOC = ""
> -
> -name = 'ubuntu_kernel_selftests'
> -
> -tests = [
> 'setup','breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','seccomp','user'
> ] - -
> -for test in tests:
> - results = job.run_test_detail('ubuntu_kernel_selftests',
> test_name=test, tag=test, timeout=60*15) -
> -# vi:set ts=4 sw=4 expandtab syntax=python:
> diff --git a/ubuntu_kernel_selftests/control.ubuntu.utopic
> b/ubuntu_kernel_selftests/control.ubuntu.utopic deleted file mode
> 100644 index 4e5b1230..00000000
> --- a/ubuntu_kernel_selftests/control.ubuntu.utopic
> +++ /dev/null
> @@ -1,21 +0,0 @@
> -AUTHOR = "Ubuntu"
> -NAME = "selftests"
> -CRITERIA = """
> -Uses built-in kernel repository self tests.
> -"""
> -SUITE = "None"
> -TIME = "SHORT"
> -TEST_CLASS = 'kernel'
> -TEST_CATEGORY = 'Functional'
> -TEST_TYPE = "client"
> -DOC = ""
> -
> -name = 'ubuntu_kernel_selftests'
> -
> -tests = [
> 'breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','user'
> ] - -
> -for test in tests:
> - results = job.run_test_detail('ubuntu_kernel_selftests',
> test_name=test, tag=test, timeout=60*15) -
> -# vi:set ts=4 sw=4 expandtab syntax=python:
> diff --git a/ubuntu_kernel_selftests/control.ubuntu.vivid
> b/ubuntu_kernel_selftests/control.ubuntu.vivid deleted file mode
> 100644 index 4e5b1230..00000000
> --- a/ubuntu_kernel_selftests/control.ubuntu.vivid
> +++ /dev/null
> @@ -1,21 +0,0 @@
> -AUTHOR = "Ubuntu"
> -NAME = "selftests"
> -CRITERIA = """
> -Uses built-in kernel repository self tests.
> -"""
> -SUITE = "None"
> -TIME = "SHORT"
> -TEST_CLASS = 'kernel'
> -TEST_CATEGORY = 'Functional'
> -TEST_TYPE = "client"
> -DOC = ""
> -
> -name = 'ubuntu_kernel_selftests'
> -
> -tests = [
> 'breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','user'
> ] - -
> -for test in tests:
> - results = job.run_test_detail('ubuntu_kernel_selftests',
> test_name=test, tag=test, timeout=60*15) -
> -# vi:set ts=4 sw=4 expandtab syntax=python:
> diff --git a/ubuntu_kernel_selftests/control.ubuntu.wily
> b/ubuntu_kernel_selftests/control.ubuntu.wily deleted file mode 100644
> index 4e5b1230..00000000
> --- a/ubuntu_kernel_selftests/control.ubuntu.wily
> +++ /dev/null
> @@ -1,21 +0,0 @@
> -AUTHOR = "Ubuntu"
> -NAME = "selftests"
> -CRITERIA = """
> -Uses built-in kernel repository self tests.
> -"""
> -SUITE = "None"
> -TIME = "SHORT"
> -TEST_CLASS = 'kernel'
> -TEST_CATEGORY = 'Functional'
> -TEST_TYPE = "client"
> -DOC = ""
> -
> -name = 'ubuntu_kernel_selftests'
> -
> -tests = [
> 'breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','user'
> ] - -
> -for test in tests:
> - results = job.run_test_detail('ubuntu_kernel_selftests',
> test_name=test, tag=test, timeout=60*15) -
> -# vi:set ts=4 sw=4 expandtab syntax=python:
> diff --git a/ubuntu_kernel_selftests/control.ubuntu.xenial
> b/ubuntu_kernel_selftests/control.ubuntu.xenial index
> ec5121c9..89436fa1 100644 ---
> a/ubuntu_kernel_selftests/control.ubuntu.xenial +++
> b/ubuntu_kernel_selftests/control.ubuntu.xenial @@ -1,5 +1,5 @@
> AUTHOR = "Ubuntu"
> -NAME = "selftests"
> +NAME = 'ubuntu_kernel_selftests'
> CRITERIA = """
> Uses built-in kernel repository self tests.
> """
> @@ -10,12 +10,24 @@ TEST_CATEGORY = 'Functional'
> TEST_TYPE = "client"
> DOC = ""
>
> -name = 'ubuntu_kernel_selftests'
>
> -tests = [
> 'setup','breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','seccomp','user'
> ] +categories = ['breakpoints', 'cpu-hotplug', 'efivarfs',
> 'memory-hotplug', 'mount', 'net', 'ptrace', 'powerpc', 'seccomp',
> 'user'] - -for test in tests:
> - results = job.run_test_detail('ubuntu_kernel_selftests',
> test_name=test, tag=test, timeout=60*15) +result =
> job.run_test_detail(NAME, test_name='setup', tag='setup',
> timeout=60*30) +if result == 'ERROR':
> + print("ERROR: test failed to build, skipping all the sub tests")
> +else:
> + for category in categories:
> + build = '{}-build'.format(category)
> + job.run_test_detail(NAME, test_name=build, tag=build,
> timeout=60*10)
> + mk_helper = os.path.join(job.testdir, NAME, 'helper.mk')
> + dir_src = os.path.join(job.bindir, 'tmp', NAME, 'src',
> 'linux/tools/testing/selftests/', category)
> + mk_src = os.path.join(dir_src, 'Makefile')
> + os.chdir(dir_src)
> + cmd = 'make -f {} -f {} gettests'.format(mk_helper, mk_src)
> + tests = utils.system_output(cmd).split()
> + for item in tests:
> + test = "{}:{}".format(category, os.path.basename(item))
> + job.run_test_detail(NAME, test_name=test, tag=test,
> timeout=60*20)
> # vi:set ts=4 sw=4 expandtab syntax=python:
> diff --git a/ubuntu_kernel_selftests/control.ubuntu.yakkety
> b/ubuntu_kernel_selftests/control.ubuntu.yakkety deleted file mode
> 100644 index 085aa7f3..00000000
> --- a/ubuntu_kernel_selftests/control.ubuntu.yakkety
> +++ /dev/null
> @@ -1,21 +0,0 @@
> -AUTHOR = "Ubuntu"
> -NAME = "selftests"
> -CRITERIA = """
> -Uses built-in kernel repository self tests.
> -"""
> -SUITE = "None"
> -TIME = "SHORT"
> -TEST_CLASS = 'kernel'
> -TEST_CATEGORY = 'Functional'
> -TEST_TYPE = "client"
> -DOC = ""
> -
> -name = 'ubuntu_kernel_selftests'
> -
> -tests = [
> 'breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','seccomp','user'
> ] - -
> -for test in tests:
> - results = job.run_test_detail('ubuntu_kernel_selftests',
> test_name=test, tag=test, timeout=60*15) -
> -# vi:set ts=4 sw=4 expandtab syntax=python:
> diff --git a/ubuntu_kernel_selftests/control.ubuntu.zesty
> b/ubuntu_kernel_selftests/control.ubuntu.zesty deleted file mode
> 100644 index ec5121c9..00000000
> --- a/ubuntu_kernel_selftests/control.ubuntu.zesty
> +++ /dev/null
> @@ -1,21 +0,0 @@
> -AUTHOR = "Ubuntu"
> -NAME = "selftests"
> -CRITERIA = """
> -Uses built-in kernel repository self tests.
> -"""
> -SUITE = "None"
> -TIME = "SHORT"
> -TEST_CLASS = 'kernel'
> -TEST_CATEGORY = 'Functional'
> -TEST_TYPE = "client"
> -DOC = ""
> -
> -name = 'ubuntu_kernel_selftests'
> -
> -tests = [
> 'setup','breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','seccomp','user'
> ] - -
> -for test in tests:
> - results = job.run_test_detail('ubuntu_kernel_selftests',
> test_name=test, tag=test, timeout=60*15) -
> -# vi:set ts=4 sw=4 expandtab syntax=python:
> diff --git a/ubuntu_kernel_selftests/helper.mk
> b/ubuntu_kernel_selftests/helper.mk new file mode 100755
> index 00000000..85fbd0ab
> --- /dev/null
> +++ b/ubuntu_kernel_selftests/helper.mk
> @@ -0,0 +1,2 @@
> +gettests:
> + @echo '$(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) $(TEST_PROGS)'
> diff --git a/ubuntu_kernel_selftests/ubuntu_kernel_selftests.py
> b/ubuntu_kernel_selftests/ubuntu_kernel_selftests.py index
> e6f98082..3f06838f 100644 ---
> a/ubuntu_kernel_selftests/ubuntu_kernel_selftests.py +++
> b/ubuntu_kernel_selftests/ubuntu_kernel_selftests.py @@ -10,6 +10,7
> @@ class ubuntu_kernel_selftests(test.test): version = 1
>
> def install_required_pkgs(self):
> + '''Function to install necessary packages.'''
> pkgs = [
> 'bc',
> 'build-essential',
> @@ -24,7 +25,7 @@ class ubuntu_kernel_selftests(test.test):
> 'pkg-config',
> 'uuid-runtime'
> ]
> - if not (self.arch == 's390x' and self.series in ['precise',
> 'trusty', 'vivid', 'xenial']):
> + if not (self.arch == 's390x' and self.series in ['trusty',
> 'xenial']): pkgs.append('libnuma-dev')
> pkgs.append('libfuse-dev')
> gcc = 'gcc' if self.arch in ['ppc64le', 'aarch64', 's390x',
> 'riscv64'] else 'gcc-multilib' @@ -49,7 +50,7 @@ class
> ubuntu_kernel_selftests(test.test): pkgs.extend(['clang', 'llvm'])
>
> cmd = 'yes "" | DEBIAN_FRONTEND=noninteractive apt-get
> install --yes --force-yes ' + ' '.join(pkgs)
> - self.results = utils.system_output(cmd, retain_output=True)
> + utils.system_output(cmd, retain_output=True)
>
> def initialize(self):
> self.arch = platform.processor()
> @@ -61,26 +62,20 @@ class ubuntu_kernel_selftests(test.test):
> self.series = distro.codename()
> self.kv = platform.release().split(".")[:2]
> self.kv = int(self.kv[0]) * 100 + int(self.kv[1])
> - pass
>
> def download(self):
> + '''Function to download kernel source.'''
> cmd = "dpkg -S /lib/modules/" + platform.release() +
> "/kernel | cut -d: -f 1 | cut -d, -f 1" pkg =
> os.popen(cmd).readlines()[0].strip() utils.system("apt-get source
> --download-only " + pkg)
> def extract(self):
> + '''Function to extract kernel source.'''
> os.system("rm -rf linux/")
> utils.system("dpkg-source -x linux*dsc linux")
>
> - def summary(self, pattern):
> - failures = list(re.finditer(pattern, self.results))
> - if failures:
> - for i in failures:
> - print('Sub test case: {}
> failed.'.format(i.group('case')))
> - return True
> - return False
> -
> def setup(self):
> + '''Function to setup the test environment.'''
> self.install_required_pkgs()
> self.job.require_gcc()
> os.chdir(self.srcdir)
> @@ -103,8 +98,8 @@ class ubuntu_kernel_selftests(test.test):
> #
> fn =
> 'linux/tools/testing/selftests/breakpoints/step_after_suspend_test.c'
> if os.path.exists(fn):
> - cmd = 'sed -i "s/tv_sec = 5;/tv_sec = 30;/" ' + fn
> - utils.system(cmd)
> + cmd = 'sed -i "s/tv_sec = 5;/tv_sec = 30;/" ' + fn
> + utils.system(cmd)
> # currently disable step_after_suspend_test as this
> breaks ssh'd login # connections to the test VMs and real H/W
> fn = 'linux/tools/testing/selftests/breakpoints/Makefile'
> @@ -136,12 +131,12 @@ class ubuntu_kernel_selftests(test.test):
> # update fix CPU hotplug test, new and old versions
> #
> print("Updating CPU hotplug test")
> -
> fn="linux/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh"
> + fn =
> "linux/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh" if
> os.path.exists(fn) and 'present_cpus=' not in open(fn).read(): cmd =
> 'cp %s/cpu-on-off-test.sh %s' % (self.bindir, fn) utils.system(cmd)
> else:
> -
> fn="linux/tools/testing/selftests/cpu-hotplug/on-off-test.sh"
> + fn =
> "linux/tools/testing/selftests/cpu-hotplug/on-off-test.sh" if
> os.path.exists(fn) and 'present_cpus=' not in open(fn).read(): cmd =
> 'cp %s/cpu-on-off-test.sh %s' % (self.bindir, fn) utils.system(cmd)
> @@ -216,40 +211,45 @@ class ubuntu_kernel_selftests(test.test):
> def run_once(self, test_name):
> if test_name == 'setup':
> return
> + if test_name.endswith('-build'):
> + os.chdir(self.srcdir)
> + if "net" in test_name:
> + cmd = "sh -c 'echo 1 >
> /proc/sys/net/ipv4/conf/all/accept_local'"
> + utils.system(cmd)
> + if self.kv >= 415:
> + # net selftests use a module built by bpf
> selftests, bpf is available since bionic kernel
> + if self.kv == 506:
> + os.environ["CLANG"] = "clang-10"
> + os.environ["LLC"] = "llc-10"
> + os.environ["LLVM_OBJCOPY"] =
> "llvm-objcopy-10"
> + os.environ["LLVM_READELF"] =
> "llvm-readelf-10"
> + elif self.kv in [504, 503]:
> + os.environ["CLANG"] = "clang-9"
> + os.environ["LLC"] = "llc-9"
> + os.environ["LLVM_OBJCOPY"] = "llvm-objcopy-9"
> + os.environ["LLVM_READELF"] = "llvm-readelf-9"
> + cmd = "make -C linux/tools/testing/selftests
> TARGETS=bpf SKIP_TARGETS=
> KDIR=/usr/src/linux-headers-{}".format(platform.release())
> + # keep running selftests/net, even if
> selftests/bpf build fails
> + utils.system(cmd, ignore_status=True)
> + cmd = "make -C linux/tools/testing/selftests
> TARGETS={}".format(test_name.replace('-build', ''))
> + utils.system_output(cmd, retain_output=True)
> + return
>
> - cmd = "sudo sh -c 'echo 1 >
> /proc/sys/net/ipv4/conf/all/accept_local'"
> - utils.system(cmd)
> -
> - os.chdir(self.srcdir)
> - if test_name == "net" and self.kv >= 415:
> - # net selftests use a module built by bpf selftests, bpf
> is available since bionic kernel
> - if self.kv == 506:
> - os.environ["CLANG"] = "clang-10"
> - os.environ["LLC"] = "llc-10"
> - os.environ["LLVM_OBJCOPY"] = "llvm-objcopy-10"
> - os.environ["LLVM_READELF"] = "llvm-readelf-10"
> - elif self.kv in [504, 503]:
> - os.environ["CLANG"] = "clang-9"
> - os.environ["LLC"] = "llc-9"
> - os.environ["LLVM_OBJCOPY"] = "llvm-objcopy-9"
> - os.environ["LLVM_READELF"] = "llvm-readelf-9"
> - cmd = "make -C linux/tools/testing/selftests TARGETS=bpf
> SKIP_TARGETS=
> KDIR=/usr/src/linux-headers-{}".format(platform.release())
> - # keep running selftests/net, even if selftests/bpf
> build fails
> - utils.system(cmd, ignore_status=True)
> - cmd = "sudo make -C linux/tools/testing/selftests TARGETS=%s
> run_tests" % test_name
> - self.results = utils.system_output(cmd, retain_output=True)
> -
> - print('========== Summary ===========')
> + category = test_name.split(':')[0]
> + sub_test = test_name.split(':')[1]
> + dir_root = os.path.join(self.srcdir, 'linux', 'tools',
> 'testing', 'selftests')
> + os.chdir(dir_root)
> + cmd = "make run_tests -C {} TEST_PROGS={} TEST_GEN_PROGS=''
> TEST_CUSTOM_PROGS=''".format(category, sub_test)
> + result = utils.system_output(cmd, retain_output=True)
>
> # Old pattern for Xenial
> pattern = re.compile('selftests: *(?P<case>[\w\-\.]+)
> \[FAIL\]\n')
> - if self.summary(pattern):
> - raise error.TestError('Test failed for ' + test_name)
> + if re.search(pattern, result):
> + raise error.TestError(test_name + ' failed.')
> # If the test was not end by previous check, check again
> with new pattern
> - pattern = re.compile('not ok [\d\.]* selftests: *({}.*:
> )?(?P<case>[\w\-\.]+)(?!.*SKIP)'.format(test_name))
> - if self.summary(pattern):
> - raise error.TestError('Test failed for ' + test_name)
> + pattern = re.compile('not ok [\d\.]* selftests: {}: {} #
> (?!.*SKIP)'.format(category, sub_test))
> + if re.search(pattern, result):
> + raise error.TestError(test_name + ' failed.')
>
> - print('No failed cases reported')
>
> # vi:set ts=4 sw=4 expandtab syntax=python:
More information about the kernel-team
mailing list