mirror of
https://github.com/samba-team/samba.git
synced 2024-12-22 13:34:15 +03:00
selftest: Add --flapping argument to filter-subunit.
This commit is contained in:
parent
5643d37a00
commit
e62654578b
@ -17,9 +17,11 @@ parser = optparse.OptionParser("filter-subunit [options] < instream > outstream"
|
||||
parser.add_option("--expected-failures", type="string",
|
||||
help="File containing list of regexes matching tests to consider known "
|
||||
"failures")
|
||||
parser.add_option("--strip-passed-output", action="store_true",
|
||||
parser.add_option("--flapping", type="string",
|
||||
help="File containing list of flapping tests, of which to ignore results.")
|
||||
parser.add_option("--strip-passed-output", action="store_true",
|
||||
help="Whether to strip output from tests that passed")
|
||||
parser.add_option("--fail-immediately", action="store_true",
|
||||
parser.add_option("--fail-immediately", action="store_true",
|
||||
help="Whether to stop on the first error", default=False)
|
||||
parser.add_option("--prefix", type="string",
|
||||
help="Add prefix to all test names")
|
||||
@ -42,6 +44,12 @@ if opts.expected_failures:
|
||||
else:
|
||||
expected_failures = {}
|
||||
|
||||
|
||||
if opts.flapping:
|
||||
flapping = subunithelper.read_test_regexes(opts.flapping)
|
||||
else:
|
||||
flapping = []
|
||||
|
||||
statistics = {
|
||||
'TESTS_UNEXPECTED_OK': 0,
|
||||
'TESTS_EXPECTED_OK': 0,
|
||||
@ -58,7 +66,8 @@ signal.signal(signal.SIGINT, handle_sigint)
|
||||
out = subunithelper.SubunitOps(sys.stdout)
|
||||
msg_ops = subunithelper.FilterOps(out, opts.prefix, expected_failures,
|
||||
opts.strip_passed_output,
|
||||
fail_immediately=opts.fail_immediately)
|
||||
fail_immediately=opts.fail_immediately,
|
||||
flapping=flapping)
|
||||
|
||||
try:
|
||||
ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
|
||||
|
@ -288,6 +288,8 @@ class FilterOps(testtools.testresult.TestResult):
|
||||
def addFailure(self, test, details=None):
|
||||
test = self._add_prefix(test)
|
||||
xfail_reason = find_in_list(self.expected_failures, test.id())
|
||||
if xfail_reason is None:
|
||||
xfail_reason = find_in_list(self.flapping, test.id())
|
||||
if xfail_reason is not None:
|
||||
self.xfail_added+=1
|
||||
self.total_xfail+=1
|
||||
@ -308,7 +310,21 @@ class FilterOps(testtools.testresult.TestResult):
|
||||
|
||||
def addSuccess(self, test, details=None):
|
||||
test = self._add_prefix(test)
|
||||
self._ops.addSuccess(test, details)
|
||||
xfail_reason = find_in_list(self.expected_failures, test.id())
|
||||
if xfail_reason is not None:
|
||||
self.uxsuccess_added += 1
|
||||
self.total_uxsuccess += 1
|
||||
if details is not None:
|
||||
details = subunit.RemoteError(unicode(details[1]) + xfail_reason.decode("utf-8"))
|
||||
else:
|
||||
details = subunit.RemoteError(xfail_reason.decode("utf-8"))
|
||||
self._ops.addUnexpectedSuccess(test, details)
|
||||
if self.output:
|
||||
self._ops.output_msg(self.output)
|
||||
if self.fail_immediately:
|
||||
raise ImmediateFail()
|
||||
else:
|
||||
self._ops.addSuccess(test, details)
|
||||
self.output = None
|
||||
|
||||
def skip_testsuite(self, name, reason=None):
|
||||
@ -319,6 +335,7 @@ class FilterOps(testtools.testresult.TestResult):
|
||||
self.error_added = 0
|
||||
self.fail_added = 0
|
||||
self.xfail_added = 0
|
||||
self.uxsuccess_added = 0
|
||||
|
||||
def end_testsuite(self, name, result, reason=None):
|
||||
xfail = False
|
||||
@ -346,7 +363,8 @@ class FilterOps(testtools.testresult.TestResult):
|
||||
self._ops.end_testsuite(name, result, reason)
|
||||
|
||||
def __init__(self, out, prefix=None, expected_failures=None,
|
||||
strip_ok_output=False, fail_immediately=False):
|
||||
strip_ok_output=False, fail_immediately=False,
|
||||
flapping=None):
|
||||
self._ops = out
|
||||
self.seen_output = False
|
||||
self.output = None
|
||||
@ -355,12 +373,18 @@ class FilterOps(testtools.testresult.TestResult):
|
||||
self.expected_failures = expected_failures
|
||||
else:
|
||||
self.expected_failures = {}
|
||||
if flapping is not None:
|
||||
self.flapping = flapping
|
||||
else:
|
||||
self.flapping = {}
|
||||
self.strip_ok_output = strip_ok_output
|
||||
self.xfail_added = 0
|
||||
self.fail_added = 0
|
||||
self.uxsuccess_added = 0
|
||||
self.total_xfail = 0
|
||||
self.total_error = 0
|
||||
self.total_fail = 0
|
||||
self.total_uxsuccess = 0
|
||||
self.error_added = 0
|
||||
self.fail_immediately = fail_immediately
|
||||
|
||||
|
@ -88,7 +88,7 @@ def cmd_testonly(opt):
|
||||
env.SUBUNIT_FORMATTER = os.getenv('SUBUNIT_FORMATTER')
|
||||
if not env.SUBUNIT_FORMATTER:
|
||||
env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate'
|
||||
env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --expected-failure=${srcdir}/selftest/flapping'
|
||||
env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --flapping=${srcdir}/selftest/flapping'
|
||||
|
||||
if Options.options.FAIL_IMMEDIATELY:
|
||||
env.FILTER_XFAIL += ' --fail-immediately'
|
||||
|
@ -1,6 +1,6 @@
|
||||
#!/bin/sh
|
||||
|
||||
FILTER_XFAIL="${PYTHON} -u ${SELFTESTDIR}/filter-subunit --expected-failures=${SELFTESTDIR}/knownfail --expected-failures=${SELFTESTDIR}/flapping"
|
||||
FILTER_XFAIL="${PYTHON} -u ${SELFTESTDIR}/filter-subunit --expected-failures=${SELFTESTDIR}/knownfail --flapping=${SELFTESTDIR}/flapping"
|
||||
if [ "x${SUBUNIT_FORMATTER}" = x"" ]; then
|
||||
SUBUNIT_FORMATTER="${PYTHON} -u ${SELFTESTDIR}/format-subunit --prefix=${SELFTESTPREFIX} --immediate"
|
||||
fi
|
||||
|
Loading…
Reference in New Issue
Block a user