[ACT][PATCH] UBUNTU: SAUCE: ubuntu_kernel_selftests: granularity improvement
Po-Hsu Lin
po-hsu.lin at canonical.com
Tue Nov 9 08:32:55 UTC 2021
BugLink: https://bugs.launchpad.net/bugs/1941052
This patch will change how we run those test cases in the selftests
directory of a kernel tree:
* Test will now abort early if the setup() has failed
* Each test suite will be built first with a SUITE_NAME-build job,
we will be able to distinguish build failures and actual test
failures more easily.
* Each test inside a suite will be executed one-by-one with name in
a format of SUITE:CASE, e.g. net:test_bpf.sh, this will help us
to improve our result hinting process, make the test report
become more reliable and easier to gate new regressions.
* Use a helper.mk file to get the executable test names, by doing
so we can workaround the issue caused by kernel source code
difference across various releases.
* Remove the summary section, we won't have a summary for a whole
suite in the end of the test. I think this feature can be added
back if needed.
* Remove out-dated control files.
* Trusty control file is untouched as this test is blacklisted.
This patch has been tested on KVM instances with X/F/H/I kernel on our
dev jenkins and X-4.15/B-4.15/B-5.4/F/H/I Oracle kernel with SRU cycle
sru-20211018, the overall result is looking good. Bionic kernel will
need fix in bug 1949889 to land first to make net tests working
properly.
Signed-off-by: Po-Hsu Lin <po-hsu.lin at canonical.com>
---
ubuntu_kernel_selftests/control | 28 ++++--
ubuntu_kernel_selftests/control.ubuntu.artful | 21 -----
ubuntu_kernel_selftests/control.ubuntu.utopic | 21 -----
ubuntu_kernel_selftests/control.ubuntu.vivid | 21 -----
ubuntu_kernel_selftests/control.ubuntu.wily | 21 -----
ubuntu_kernel_selftests/control.ubuntu.xenial | 24 +++--
.../control.ubuntu.yakkety | 21 -----
ubuntu_kernel_selftests/control.ubuntu.zesty | 21 -----
ubuntu_kernel_selftests/helper.mk | 2 +
.../ubuntu_kernel_selftests.py | 88 +++++++++----------
10 files changed, 84 insertions(+), 184 deletions(-)
delete mode 100644 ubuntu_kernel_selftests/control.ubuntu.artful
delete mode 100644 ubuntu_kernel_selftests/control.ubuntu.utopic
delete mode 100644 ubuntu_kernel_selftests/control.ubuntu.vivid
delete mode 100644 ubuntu_kernel_selftests/control.ubuntu.wily
delete mode 100644 ubuntu_kernel_selftests/control.ubuntu.yakkety
delete mode 100644 ubuntu_kernel_selftests/control.ubuntu.zesty
create mode 100755 ubuntu_kernel_selftests/helper.mk
diff --git a/ubuntu_kernel_selftests/control b/ubuntu_kernel_selftests/control
index e2874196..4fcee658 100644
--- a/ubuntu_kernel_selftests/control
+++ b/ubuntu_kernel_selftests/control
@@ -1,5 +1,5 @@
AUTHOR = "Ubuntu"
-NAME = "selftests"
+NAME = 'ubuntu_kernel_selftests'
CRITERIA = """
Uses built-in kernel repository self tests.
"""
@@ -10,20 +10,32 @@ TEST_CATEGORY = 'Functional'
TEST_TYPE = "client"
DOC = ""
-name = 'ubuntu_kernel_selftests'
-
-tests = [ 'setup','breakpoints','cpu-hotplug','efivarfs','memfd','memory-hotplug','mount','net','ptrace','seccomp','timers','powerpc','user','ftrace' ]
+categories = ['breakpoints', 'cpu-hotplug', 'efivarfs', 'memfd', 'memory-hotplug', 'mount', 'net', 'ptrace', 'seccomp', 'timers', 'powerpc', 'user', 'ftrace']
#
# The seccomp tests on 4.19+ on non-x86 are known to be fail and
# need fixing up. For now, disable them.
#
release = platform.release().split(".")[:2]
release = int(release[0])*100 + int(release[1])
-if 'seccomp' in tests and release > 418 and platform.machine() not in ['x86_64', 'ppc64le']:
- tests.remove('seccomp')
+if 'seccomp' in categories and release > 418 and platform.machine() not in ['x86_64', 'ppc64le']:
+ categories.remove('seccomp')
-for test in tests:
- results = job.run_test_detail('ubuntu_kernel_selftests', test_name=test, tag=test, timeout=60*30)
+result = job.run_test_detail(NAME, test_name='setup', tag='setup', timeout=60*30)
+if result == 'ERROR':
+ print("ERROR: test failed to build, skipping all the sub tests")
+else:
+ for category in categories:
+ build = '{}-build'.format(category)
+ job.run_test_detail(NAME, test_name=build, tag=build, timeout=60*10)
+ mk_helper = os.path.join(job.testdir, NAME, 'helper.mk')
+ dir_src = os.path.join(job.bindir, 'tmp', NAME, 'src', 'linux/tools/testing/selftests/', category)
+ mk_src = os.path.join(dir_src, 'Makefile')
+ os.chdir(dir_src)
+ cmd = 'make -f {} -f {} gettests'.format(mk_helper, mk_src)
+ tests = utils.system_output(cmd).split()
+ for item in tests:
+ test = "{}:{}".format(category, os.path.basename(item))
+ job.run_test_detail(NAME, test_name=test, tag=test, timeout=60*20)
# vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_kernel_selftests/control.ubuntu.artful b/ubuntu_kernel_selftests/control.ubuntu.artful
deleted file mode 100644
index ec5121c9..00000000
--- a/ubuntu_kernel_selftests/control.ubuntu.artful
+++ /dev/null
@@ -1,21 +0,0 @@
-AUTHOR = "Ubuntu"
-NAME = "selftests"
-CRITERIA = """
-Uses built-in kernel repository self tests.
-"""
-SUITE = "None"
-TIME = "SHORT"
-TEST_CLASS = 'kernel'
-TEST_CATEGORY = 'Functional'
-TEST_TYPE = "client"
-DOC = ""
-
-name = 'ubuntu_kernel_selftests'
-
-tests = [ 'setup','breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','seccomp','user' ]
-
-
-for test in tests:
- results = job.run_test_detail('ubuntu_kernel_selftests', test_name=test, tag=test, timeout=60*15)
-
-# vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_kernel_selftests/control.ubuntu.utopic b/ubuntu_kernel_selftests/control.ubuntu.utopic
deleted file mode 100644
index 4e5b1230..00000000
--- a/ubuntu_kernel_selftests/control.ubuntu.utopic
+++ /dev/null
@@ -1,21 +0,0 @@
-AUTHOR = "Ubuntu"
-NAME = "selftests"
-CRITERIA = """
-Uses built-in kernel repository self tests.
-"""
-SUITE = "None"
-TIME = "SHORT"
-TEST_CLASS = 'kernel'
-TEST_CATEGORY = 'Functional'
-TEST_TYPE = "client"
-DOC = ""
-
-name = 'ubuntu_kernel_selftests'
-
-tests = [ 'breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','user' ]
-
-
-for test in tests:
- results = job.run_test_detail('ubuntu_kernel_selftests', test_name=test, tag=test, timeout=60*15)
-
-# vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_kernel_selftests/control.ubuntu.vivid b/ubuntu_kernel_selftests/control.ubuntu.vivid
deleted file mode 100644
index 4e5b1230..00000000
--- a/ubuntu_kernel_selftests/control.ubuntu.vivid
+++ /dev/null
@@ -1,21 +0,0 @@
-AUTHOR = "Ubuntu"
-NAME = "selftests"
-CRITERIA = """
-Uses built-in kernel repository self tests.
-"""
-SUITE = "None"
-TIME = "SHORT"
-TEST_CLASS = 'kernel'
-TEST_CATEGORY = 'Functional'
-TEST_TYPE = "client"
-DOC = ""
-
-name = 'ubuntu_kernel_selftests'
-
-tests = [ 'breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','user' ]
-
-
-for test in tests:
- results = job.run_test_detail('ubuntu_kernel_selftests', test_name=test, tag=test, timeout=60*15)
-
-# vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_kernel_selftests/control.ubuntu.wily b/ubuntu_kernel_selftests/control.ubuntu.wily
deleted file mode 100644
index 4e5b1230..00000000
--- a/ubuntu_kernel_selftests/control.ubuntu.wily
+++ /dev/null
@@ -1,21 +0,0 @@
-AUTHOR = "Ubuntu"
-NAME = "selftests"
-CRITERIA = """
-Uses built-in kernel repository self tests.
-"""
-SUITE = "None"
-TIME = "SHORT"
-TEST_CLASS = 'kernel'
-TEST_CATEGORY = 'Functional'
-TEST_TYPE = "client"
-DOC = ""
-
-name = 'ubuntu_kernel_selftests'
-
-tests = [ 'breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','user' ]
-
-
-for test in tests:
- results = job.run_test_detail('ubuntu_kernel_selftests', test_name=test, tag=test, timeout=60*15)
-
-# vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_kernel_selftests/control.ubuntu.xenial b/ubuntu_kernel_selftests/control.ubuntu.xenial
index ec5121c9..89436fa1 100644
--- a/ubuntu_kernel_selftests/control.ubuntu.xenial
+++ b/ubuntu_kernel_selftests/control.ubuntu.xenial
@@ -1,5 +1,5 @@
AUTHOR = "Ubuntu"
-NAME = "selftests"
+NAME = 'ubuntu_kernel_selftests'
CRITERIA = """
Uses built-in kernel repository self tests.
"""
@@ -10,12 +10,24 @@ TEST_CATEGORY = 'Functional'
TEST_TYPE = "client"
DOC = ""
-name = 'ubuntu_kernel_selftests'
-tests = [ 'setup','breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','seccomp','user' ]
+categories = ['breakpoints', 'cpu-hotplug', 'efivarfs', 'memory-hotplug', 'mount', 'net', 'ptrace', 'powerpc', 'seccomp', 'user']
-
-for test in tests:
- results = job.run_test_detail('ubuntu_kernel_selftests', test_name=test, tag=test, timeout=60*15)
+result = job.run_test_detail(NAME, test_name='setup', tag='setup', timeout=60*30)
+if result == 'ERROR':
+ print("ERROR: test failed to build, skipping all the sub tests")
+else:
+ for category in categories:
+ build = '{}-build'.format(category)
+ job.run_test_detail(NAME, test_name=build, tag=build, timeout=60*10)
+ mk_helper = os.path.join(job.testdir, NAME, 'helper.mk')
+ dir_src = os.path.join(job.bindir, 'tmp', NAME, 'src', 'linux/tools/testing/selftests/', category)
+ mk_src = os.path.join(dir_src, 'Makefile')
+ os.chdir(dir_src)
+ cmd = 'make -f {} -f {} gettests'.format(mk_helper, mk_src)
+ tests = utils.system_output(cmd).split()
+ for item in tests:
+ test = "{}:{}".format(category, os.path.basename(item))
+ job.run_test_detail(NAME, test_name=test, tag=test, timeout=60*20)
# vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_kernel_selftests/control.ubuntu.yakkety b/ubuntu_kernel_selftests/control.ubuntu.yakkety
deleted file mode 100644
index 085aa7f3..00000000
--- a/ubuntu_kernel_selftests/control.ubuntu.yakkety
+++ /dev/null
@@ -1,21 +0,0 @@
-AUTHOR = "Ubuntu"
-NAME = "selftests"
-CRITERIA = """
-Uses built-in kernel repository self tests.
-"""
-SUITE = "None"
-TIME = "SHORT"
-TEST_CLASS = 'kernel'
-TEST_CATEGORY = 'Functional'
-TEST_TYPE = "client"
-DOC = ""
-
-name = 'ubuntu_kernel_selftests'
-
-tests = [ 'breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','seccomp','user' ]
-
-
-for test in tests:
- results = job.run_test_detail('ubuntu_kernel_selftests', test_name=test, tag=test, timeout=60*15)
-
-# vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_kernel_selftests/control.ubuntu.zesty b/ubuntu_kernel_selftests/control.ubuntu.zesty
deleted file mode 100644
index ec5121c9..00000000
--- a/ubuntu_kernel_selftests/control.ubuntu.zesty
+++ /dev/null
@@ -1,21 +0,0 @@
-AUTHOR = "Ubuntu"
-NAME = "selftests"
-CRITERIA = """
-Uses built-in kernel repository self tests.
-"""
-SUITE = "None"
-TIME = "SHORT"
-TEST_CLASS = 'kernel'
-TEST_CATEGORY = 'Functional'
-TEST_TYPE = "client"
-DOC = ""
-
-name = 'ubuntu_kernel_selftests'
-
-tests = [ 'setup','breakpoints','cpu-hotplug','efivarfs','memory-hotplug','mount','net','ptrace','powerpc','seccomp','user' ]
-
-
-for test in tests:
- results = job.run_test_detail('ubuntu_kernel_selftests', test_name=test, tag=test, timeout=60*15)
-
-# vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_kernel_selftests/helper.mk b/ubuntu_kernel_selftests/helper.mk
new file mode 100755
index 00000000..85fbd0ab
--- /dev/null
+++ b/ubuntu_kernel_selftests/helper.mk
@@ -0,0 +1,2 @@
+gettests:
+ @echo '$(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) $(TEST_PROGS)'
diff --git a/ubuntu_kernel_selftests/ubuntu_kernel_selftests.py b/ubuntu_kernel_selftests/ubuntu_kernel_selftests.py
index e6f98082..3f06838f 100644
--- a/ubuntu_kernel_selftests/ubuntu_kernel_selftests.py
+++ b/ubuntu_kernel_selftests/ubuntu_kernel_selftests.py
@@ -10,6 +10,7 @@ class ubuntu_kernel_selftests(test.test):
version = 1
def install_required_pkgs(self):
+ '''Function to install necessary packages.'''
pkgs = [
'bc',
'build-essential',
@@ -24,7 +25,7 @@ class ubuntu_kernel_selftests(test.test):
'pkg-config',
'uuid-runtime'
]
- if not (self.arch == 's390x' and self.series in ['precise', 'trusty', 'vivid', 'xenial']):
+ if not (self.arch == 's390x' and self.series in ['trusty', 'xenial']):
pkgs.append('libnuma-dev')
pkgs.append('libfuse-dev')
gcc = 'gcc' if self.arch in ['ppc64le', 'aarch64', 's390x', 'riscv64'] else 'gcc-multilib'
@@ -49,7 +50,7 @@ class ubuntu_kernel_selftests(test.test):
pkgs.extend(['clang', 'llvm'])
cmd = 'yes "" | DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes ' + ' '.join(pkgs)
- self.results = utils.system_output(cmd, retain_output=True)
+ utils.system_output(cmd, retain_output=True)
def initialize(self):
self.arch = platform.processor()
@@ -61,26 +62,20 @@ class ubuntu_kernel_selftests(test.test):
self.series = distro.codename()
self.kv = platform.release().split(".")[:2]
self.kv = int(self.kv[0]) * 100 + int(self.kv[1])
- pass
def download(self):
+ '''Function to download kernel source.'''
cmd = "dpkg -S /lib/modules/" + platform.release() + "/kernel | cut -d: -f 1 | cut -d, -f 1"
pkg = os.popen(cmd).readlines()[0].strip()
utils.system("apt-get source --download-only " + pkg)
def extract(self):
+ '''Function to extract kernel source.'''
os.system("rm -rf linux/")
utils.system("dpkg-source -x linux*dsc linux")
- def summary(self, pattern):
- failures = list(re.finditer(pattern, self.results))
- if failures:
- for i in failures:
- print('Sub test case: {} failed.'.format(i.group('case')))
- return True
- return False
-
def setup(self):
+ '''Function to setup the test environment.'''
self.install_required_pkgs()
self.job.require_gcc()
os.chdir(self.srcdir)
@@ -103,8 +98,8 @@ class ubuntu_kernel_selftests(test.test):
#
fn = 'linux/tools/testing/selftests/breakpoints/step_after_suspend_test.c'
if os.path.exists(fn):
- cmd = 'sed -i "s/tv_sec = 5;/tv_sec = 30;/" ' + fn
- utils.system(cmd)
+ cmd = 'sed -i "s/tv_sec = 5;/tv_sec = 30;/" ' + fn
+ utils.system(cmd)
# currently disable step_after_suspend_test as this breaks ssh'd login
# connections to the test VMs and real H/W
fn = 'linux/tools/testing/selftests/breakpoints/Makefile'
@@ -136,12 +131,12 @@ class ubuntu_kernel_selftests(test.test):
# update fix CPU hotplug test, new and old versions
#
print("Updating CPU hotplug test")
- fn="linux/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh"
+ fn = "linux/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh"
if os.path.exists(fn) and 'present_cpus=' not in open(fn).read():
cmd = 'cp %s/cpu-on-off-test.sh %s' % (self.bindir, fn)
utils.system(cmd)
else:
- fn="linux/tools/testing/selftests/cpu-hotplug/on-off-test.sh"
+ fn = "linux/tools/testing/selftests/cpu-hotplug/on-off-test.sh"
if os.path.exists(fn) and 'present_cpus=' not in open(fn).read():
cmd = 'cp %s/cpu-on-off-test.sh %s' % (self.bindir, fn)
utils.system(cmd)
@@ -216,40 +211,45 @@ class ubuntu_kernel_selftests(test.test):
def run_once(self, test_name):
if test_name == 'setup':
return
+ if test_name.endswith('-build'):
+ os.chdir(self.srcdir)
+ if "net" in test_name:
+ cmd = "sh -c 'echo 1 > /proc/sys/net/ipv4/conf/all/accept_local'"
+ utils.system(cmd)
+ if self.kv >= 415:
+ # net selftests use a module built by bpf selftests, bpf is available since bionic kernel
+ if self.kv == 506:
+ os.environ["CLANG"] = "clang-10"
+ os.environ["LLC"] = "llc-10"
+ os.environ["LLVM_OBJCOPY"] = "llvm-objcopy-10"
+ os.environ["LLVM_READELF"] = "llvm-readelf-10"
+ elif self.kv in [504, 503]:
+ os.environ["CLANG"] = "clang-9"
+ os.environ["LLC"] = "llc-9"
+ os.environ["LLVM_OBJCOPY"] = "llvm-objcopy-9"
+ os.environ["LLVM_READELF"] = "llvm-readelf-9"
+ cmd = "make -C linux/tools/testing/selftests TARGETS=bpf SKIP_TARGETS= KDIR=/usr/src/linux-headers-{}".format(platform.release())
+ # keep running selftests/net, even if selftests/bpf build fails
+ utils.system(cmd, ignore_status=True)
+ cmd = "make -C linux/tools/testing/selftests TARGETS={}".format(test_name.replace('-build', ''))
+ utils.system_output(cmd, retain_output=True)
+ return
- cmd = "sudo sh -c 'echo 1 > /proc/sys/net/ipv4/conf/all/accept_local'"
- utils.system(cmd)
-
- os.chdir(self.srcdir)
- if test_name == "net" and self.kv >= 415:
- # net selftests use a module built by bpf selftests, bpf is available since bionic kernel
- if self.kv == 506:
- os.environ["CLANG"] = "clang-10"
- os.environ["LLC"] = "llc-10"
- os.environ["LLVM_OBJCOPY"] = "llvm-objcopy-10"
- os.environ["LLVM_READELF"] = "llvm-readelf-10"
- elif self.kv in [504, 503]:
- os.environ["CLANG"] = "clang-9"
- os.environ["LLC"] = "llc-9"
- os.environ["LLVM_OBJCOPY"] = "llvm-objcopy-9"
- os.environ["LLVM_READELF"] = "llvm-readelf-9"
- cmd = "make -C linux/tools/testing/selftests TARGETS=bpf SKIP_TARGETS= KDIR=/usr/src/linux-headers-{}".format(platform.release())
- # keep running selftests/net, even if selftests/bpf build fails
- utils.system(cmd, ignore_status=True)
- cmd = "sudo make -C linux/tools/testing/selftests TARGETS=%s run_tests" % test_name
- self.results = utils.system_output(cmd, retain_output=True)
-
- print('========== Summary ===========')
+ category = test_name.split(':')[0]
+ sub_test = test_name.split(':')[1]
+ dir_root = os.path.join(self.srcdir, 'linux', 'tools', 'testing', 'selftests')
+ os.chdir(dir_root)
+ cmd = "make run_tests -C {} TEST_PROGS={} TEST_GEN_PROGS='' TEST_CUSTOM_PROGS=''".format(category, sub_test)
+ result = utils.system_output(cmd, retain_output=True)
# Old pattern for Xenial
pattern = re.compile('selftests: *(?P<case>[\w\-\.]+) \[FAIL\]\n')
- if self.summary(pattern):
- raise error.TestError('Test failed for ' + test_name)
+ if re.search(pattern, result):
+ raise error.TestError(test_name + ' failed.')
# If the test was not end by previous check, check again with new pattern
- pattern = re.compile('not ok [\d\.]* selftests: *({}.*: )?(?P<case>[\w\-\.]+)(?!.*SKIP)'.format(test_name))
- if self.summary(pattern):
- raise error.TestError('Test failed for ' + test_name)
+ pattern = re.compile('not ok [\d\.]* selftests: {}: {} # (?!.*SKIP)'.format(category, sub_test))
+ if re.search(pattern, result):
+ raise error.TestError(test_name + ' failed.')
- print('No failed cases reported')
# vi:set ts=4 sw=4 expandtab syntax=python:
--
2.25.1
More information about the kernel-team
mailing list