[ACT][PATCH] UBUNTU: SAUCE: ubuntu_ltp: improve test granularity
Po-Hsu Lin
po-hsu.lin at canonical.com
Thu Dec 2 04:58:06 UTC 2021
Improve the test granularity by:
* Running tests from different subsets one-by-one. The test case
name will be in a format of category:sub-test , e.g. mm:mtest01
* Remove the blacklist "skip" file, which was feed to runltp
directly, integrate the list into test iteration loop
* Remove the summary section
This change will make the test number change from 7 (with 7 sub-sets)
to around 280 cases. It will be more reliable and easier to hint.
Commit tested on dev jenkins with Focal kernel on a KVM instance. The
test result looks good.
Signed-off-by: Po-Hsu Lin <po-hsu.lin at canonical.com>
---
ubuntu_ltp/control | 17 ++++++++--
ubuntu_ltp/skip | 6 ----
ubuntu_ltp/ubuntu_ltp.py | 73 +++++++++++++---------------------------
3 files changed, 38 insertions(+), 58 deletions(-)
delete mode 100644 ubuntu_ltp/skip
diff --git a/ubuntu_ltp/control b/ubuntu_ltp/control
index bd5968f4..b35fc23b 100644
--- a/ubuntu_ltp/control
+++ b/ubuntu_ltp/control
@@ -14,11 +14,22 @@ result = job.run_test_detail(NAME, test_name='setup', tag='setup', timeout=40*60
if result == 'ERROR':
print("ERROR: test failed to build, skipping all the sub tests")
else:
- tests = [
+ categories = [
'fs', 'mm', 'commands',
'cpuhotplug', 'net.ipv6_lib', 'cve', 'crypto'
]
- for test in tests:
- results = job.run_test_detail(NAME, test_name=test, tag=test, timeout=60*60*2)
+ blacklist = ['oom01', 'oom02', 'oom03', 'oom04' ,'oom05', 'mkswap01_sh'] # oom* (lp:1847963), mkswap01_sh (lp:1830584)
+ for category in categories:
+ fn = '/opt/ltp/runtest/' + category
+ with open(fn , 'r') as f:
+ for line in f:
+ if line.strip() and not line.startswith('#'):
+ if line.split()[0] in blacklist:
+ continue
+ with open ('/tmp/target' , 'w') as t:
+ t.write(line)
+ testcase = '{}:{}'.format(category, line.split()[0])
+ timeout_threshold = 60*60
+ job.run_test_detail(NAME, test_name=testcase, tag=testcase, timeout=timeout_threshold)
# vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_ltp/skip b/ubuntu_ltp/skip
deleted file mode 100644
index 0624078e..00000000
--- a/ubuntu_ltp/skip
+++ /dev/null
@@ -1,6 +0,0 @@
-oom01
-oom02
-oom03
-oom04
-oom05
-mkswap01_sh mkswap01.sh
diff --git a/ubuntu_ltp/ubuntu_ltp.py b/ubuntu_ltp/ubuntu_ltp.py
index c6ee9178..16256c45 100644
--- a/ubuntu_ltp/ubuntu_ltp.py
+++ b/ubuntu_ltp/ubuntu_ltp.py
@@ -5,9 +5,7 @@ import os
import platform
import re
import shutil
-import time
from autotest.client import test, utils
-from autotest.client.shared import error
class ubuntu_ltp(test.test):
version = 1
@@ -87,8 +85,6 @@ class ubuntu_ltp(test.test):
utils.make(nprocs)
utils.make('install')
- cmd = 'cat %s > /tmp/skip' % os.path.join(self.bindir, 'skip')
- utils.system_output(cmd)
# run_once
#
@@ -97,50 +93,29 @@ class ubuntu_ltp(test.test):
def run_once(self, test_name):
if test_name == 'setup':
return
- fn = '/tmp/syscalls-' + time.strftime("%h%d-%H%M%S")
- log_failed = fn + '.failed'
- log_output = fn + '.output'
-
- fn = '/opt/ltp/runtest/%s' % (test_name)
-
- print("Setting LTP_TIMEOUT_MUL exceptions...")
- print("Setting LTP_TIMEOUT_MUL=3 for cve-2018-1000204 / ioctl_sg01 (lp:1899413, lp:1936886, lp:1949934)")
- timeout_cases = {'zram01': '5', 'ioctl_sg01': '3'}
- if self.arch in ['ppc64le']:
- print("Running on PowerPC, set timeout multiplier LTP_TIMEOUT_MUL=3 for fs_fill (lp:1878763)")
- timeout_cases['fs_fill'] = '3'
-
- with open(fn , 'r') as f:
- for line in f:
- if line.strip() and not line.startswith('#'):
- # Reset the timeout multiplier
- os.environ["LTP_TIMEOUT_MUL"] = '1'
- with open ('/tmp/target' , 'w') as t:
- t.write(line)
-
- for _case in timeout_cases:
- if _case in line:
- os.environ["LTP_TIMEOUT_MUL"] = timeout_cases[_case]
- break
-
- cmd = '/opt/ltp/runltp -f /tmp/target -S /tmp/skip -C %s -q -l %s -o %s -T /dev/null' % (log_failed, log_output, log_output)
- utils.run(cmd, ignore_status=True, verbose=False)
- # /dev/loop# creation will be taken care by the runltp
-
-
- num_failed = sum(1 for line in open(log_failed))
- print("== Test Suite Summary ==")
- print("{} test cases failed".format(num_failed))
-
- if num_failed > 0:
- cmd = "awk '{print$1}' " + log_failed + " | sort | uniq | tr '\n' ' '"
- failed_list = utils.system_output(cmd, retain_output=False, verbose=False)
- print("Failed test cases : %s" % failed_list)
-
- cmd = 'cat ' + log_output
- utils.system_output(cmd, retain_output=True, verbose=False)
-
- if num_failed > 0:
- raise error.TestError('Test failed for ' + test_name)
+
+ test_case = test_name.split(':')[1]
+
+ if test_case == 'zram01':
+ print("Setting LTP_TIMEOUT_MUL=5 for zram01 test (lp:1897556)")
+ os.environ["LTP_TIMEOUT_MUL"] = '5'
+ elif test_case in ['cve-2018-1000204 ', 'ioctl_sg01']:
+ print("Setting LTP_TIMEOUT_MUL=3 for cve-2018-1000204 / ioctl_sg01 (lp:1899413, lp:1936886, lp:1949934)")
+ os.environ["LTP_TIMEOUT_MUL"] = '3'
+ elif test_case == 'fs_fill' and self.arch == 'ppc64le':
+ print("Setting LTP_TIMEOUT_MUL=3 for fs_fill on PowerPC")
+
+ cmd = '/opt/ltp/runltp -f /tmp/target -q -C /dev/null -l /dev/null -T /dev/null'
+ print(utils.system_output(cmd, verbose=False))
+ # /dev/loop# creation will be taken care by the runltp
+
+ def cleanup(self, test_name):
+ if test_name == 'setup':
+ return
+
+ # Restore the timeout multiplier
+ if 'LTP_TIMEOUT_MUL' in os.environ:
+ print("Restore timeout multiplier LTP_TIMEOUT_MUL back to default")
+ del os.environ["LTP_TIMEOUT_MUL"]
# vi:set ts=4 sw=4 expandtab syntax=python:
--
2.25.1
More information about the kernel-team
mailing list