mirror of
https://github.com/samba-team/samba.git
synced 2025-03-29 02:50:28 +03:00
selftest: Replace perl subunit formatter with python subunit formatter,
so we can leverage the work happening in python-subunit.
This commit is contained in:
Jelmer Vernooij
parent
0c78368a31
commit
ef3fb75261
@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env python
|
||||
# vim: expandtab
|
||||
# Pretty-format subunit output
|
||||
# Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
|
||||
# Published under the GNU GPL, v3 or later
|
||||
@ -9,24 +10,179 @@ import sys
|
||||
|
||||
import subunithelper
|
||||
|
||||
class PlainFormatter(object):
|
||||
|
||||
def __init__(self, summaryfile, verbose, immediate, statistics, totaltests=None):
|
||||
self.verbose = verbose
|
||||
self.immediate = immediate
|
||||
self.statistics = statistics
|
||||
self.start_time = None
|
||||
self.test_output = {}
|
||||
self.suitesfailed = []
|
||||
self.suites_ok = 0
|
||||
self.skips = {}
|
||||
self.summaryfile = summaryfile
|
||||
self.index = 0
|
||||
self.NAME = None
|
||||
self.totalsuites = totaltests
|
||||
|
||||
def testsuite_count(self, count):
|
||||
self.totalsuites = count
|
||||
|
||||
def report_time(self, time):
|
||||
if self.start_time is None:
|
||||
self.start_time = time
|
||||
self.last_time = time
|
||||
|
||||
def start_testsuite(self, name):
|
||||
self.index += 1
|
||||
self.NAME = name
|
||||
self.START_TIME = self.last_time
|
||||
|
||||
duration = self.START_TIME - self.start_time
|
||||
|
||||
if not self.verbose:
|
||||
self.test_output[name] = ""
|
||||
|
||||
out = ""
|
||||
out += "[%d" % self.index
|
||||
if self.totalsuites is not None:
|
||||
out += "/%d" % self.totalsuites
|
||||
out += " in %ds" % duration
|
||||
if self.suitesfailed:
|
||||
out += ", %d errors" % (len(self.suitesfailed),)
|
||||
out += "] %s" % name
|
||||
if self.immediate:
|
||||
sys.stdout.write(out + "\n")
|
||||
else:
|
||||
sys.stdout.write(out + ": ")
|
||||
|
||||
def output_msg(self, output):
|
||||
if self.verbose:
|
||||
sys.stdout.write(output)
|
||||
elif self.NAME is not None:
|
||||
self.test_output[self.NAME] += output
|
||||
else:
|
||||
sys.stdout.write(output)
|
||||
|
||||
def control_msg(self, output):
|
||||
#$self->output_msg($output)
|
||||
pass
|
||||
|
||||
def end_testsuite(self, name, result, reason):
|
||||
out = ""
|
||||
unexpected = 0
|
||||
|
||||
if not name in self.test_output:
|
||||
print "no output for name[%s]" % name
|
||||
|
||||
if result in ("success", "xfail"):
|
||||
self.suites_ok+=1
|
||||
else:
|
||||
self.output_msg("ERROR: Testsuite[%s]\nREASON: %s\n" % (name, reason))
|
||||
self.suitesfailed.append(name)
|
||||
if self.immediate and not self.verbose:
|
||||
out += self.test_output[name]
|
||||
unexpected = 1
|
||||
|
||||
if not self.immediate:
|
||||
if not unexpected:
|
||||
out += " ok\n"
|
||||
else:
|
||||
out += " " + result.upper() + "\n"
|
||||
|
||||
sys.stdout.write(out)
|
||||
|
||||
def start_test(self, testname):
|
||||
pass
|
||||
|
||||
def end_test(self, testname, result, unexpected, reason):
|
||||
append = ""
|
||||
|
||||
if not unexpected:
|
||||
self.test_output[self.NAME] = ""
|
||||
if not self.immediate:
|
||||
sys.stdout.write({
|
||||
'failure': 'f',
|
||||
'xfail': 'X',
|
||||
'skip': 's',
|
||||
'success': '.'}.get(result, "?(%s)" % result))
|
||||
return
|
||||
|
||||
reason = reason.strip()
|
||||
|
||||
append = "UNEXPECTED(%s): %s\nREASON: %s\n" % (result, testname, reason)
|
||||
|
||||
self.test_output[self.NAME] += append
|
||||
|
||||
if self.immediate and not self.verbose:
|
||||
print self.test_output[self.NAME]
|
||||
self.test_output[self.NAME] = ""
|
||||
|
||||
if not self.immediate:
|
||||
sys.stdout.write({
|
||||
'error': 'E',
|
||||
'failure': 'F',
|
||||
'success': 'S'}.get(result, "?"))
|
||||
|
||||
def summary(self):
|
||||
f = open(self.summaryfile, 'w+')
|
||||
|
||||
if self.suitesfailed:
|
||||
f.write("= Failed tests =\n")
|
||||
|
||||
for suite in self.suitesfailed:
|
||||
f.write("== %s ==\n" % suite)
|
||||
f.write(self.test_output[suite]+"\n\n")
|
||||
|
||||
f.write("\n")
|
||||
|
||||
if not self.immediate and not self.verbose:
|
||||
for suite in self.suitesfailed:
|
||||
print "==============================================================================="
|
||||
print "FAIL: %s" % suite
|
||||
print self.test_output[suite]
|
||||
print ""
|
||||
|
||||
f.write("= Skipped tests =\n")
|
||||
for reason in self.skips.keys():
|
||||
f.write(reason + "\n")
|
||||
for name in self.skips[reason]:
|
||||
f.write("\t%s\n" % name)
|
||||
f.write("\n")
|
||||
f.close()
|
||||
|
||||
print "\nA summary with detailed information can be found in:\n %s\n" % self.summaryfile
|
||||
|
||||
if not self.suitesfailed:
|
||||
ok = self.statistics['TESTS_EXPECTED_OK'] + self.statistics['TESTS_EXPECTED_FAIL']
|
||||
print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
|
||||
else:
|
||||
print "\nFAILED (%d failures and %d errors in %d testsuites)" % (self.statistics['TESTS_UNEXPECTED_FAIL'], self.statistics['TESTS_ERROR'], len(self.suitesfailed))
|
||||
|
||||
def skip_testsuite(self, name, reason="UNKNOWN"):
|
||||
self.skips.setdefault(reason, []).append(name)
|
||||
if self.totalsuites:
|
||||
self.totalsuites-=1
|
||||
|
||||
parser = optparse.OptionParser("format-subunit [options]")
|
||||
parser.add_option("--verbose", action="store_true",
|
||||
help="Be verbose")
|
||||
help="Be verbose")
|
||||
parser.add_option("--immediate", action="store_true",
|
||||
help="Show failures immediately, don't wait until test run has finished")
|
||||
help="Show failures immediately, don't wait until test run has finished")
|
||||
parser.add_option("--prefix", type="string", default=".",
|
||||
help="Prefix to write summary to")
|
||||
help="Prefix to write summary to")
|
||||
|
||||
opts, args = parser.parse_args()
|
||||
|
||||
statistics = {
|
||||
'SUITES_FAIL': 0,
|
||||
'TESTS_UNEXPECTED_OK': 0,
|
||||
'TESTS_EXPECTED_OK': 0,
|
||||
'TESTS_UNEXPECTED_FAIL': 0,
|
||||
'TESTS_EXPECTED_FAIL': 0,
|
||||
'TESTS_ERROR': 0,
|
||||
'TESTS_SKIP': 0,
|
||||
'SUITES_FAIL': 0,
|
||||
'TESTS_UNEXPECTED_OK': 0,
|
||||
'TESTS_EXPECTED_OK': 0,
|
||||
'TESTS_UNEXPECTED_FAIL': 0,
|
||||
'TESTS_EXPECTED_FAIL': 0,
|
||||
'TESTS_ERROR': 0,
|
||||
'TESTS_SKIP': 0,
|
||||
}
|
||||
|
||||
msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose, opts.immediate, statistics)
|
||||
|
||||
@ -1,246 +0,0 @@
|
||||
#!/usr/bin/perl
|
||||
# Plain text output for selftest
|
||||
# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
package output::plain;
|
||||
use Exporter;
|
||||
@ISA = qw(Exporter);
|
||||
|
||||
use FindBin qw($RealBin);
|
||||
use lib "$RealBin/..";
|
||||
|
||||
use strict;
|
||||
|
||||
sub new($$$$$$$) {
|
||||
my ($class, $summaryfile, $verbose, $immediate, $statistics, $totaltests) = @_;
|
||||
my $self = {
|
||||
verbose => $verbose,
|
||||
immediate => $immediate,
|
||||
statistics => $statistics,
|
||||
start_time => undef,
|
||||
test_output => {},
|
||||
suitesfailed => [],
|
||||
suites_ok => 0,
|
||||
skips => {},
|
||||
summaryfile => $summaryfile,
|
||||
index => 0,
|
||||
totalsuites => $totaltests,
|
||||
};
|
||||
bless($self, $class);
|
||||
}
|
||||
|
||||
sub testsuite_count($$)
|
||||
{
|
||||
my ($self, $count) = @_;
|
||||
$self->{totalsuites} = $count;
|
||||
}
|
||||
|
||||
sub report_time($$)
|
||||
{
|
||||
my ($self, $time) = @_;
|
||||
unless ($self->{start_time}) {
|
||||
$self->{start_time} = $time;
|
||||
}
|
||||
$self->{last_time} = $time;
|
||||
}
|
||||
|
||||
sub output_msg($$);
|
||||
|
||||
sub start_testsuite($$)
|
||||
{
|
||||
my ($self, $name) = @_;
|
||||
|
||||
$self->{index}++;
|
||||
$self->{NAME} = $name;
|
||||
$self->{START_TIME} = $self->{last_time};
|
||||
|
||||
my $duration = $self->{START_TIME} - $self->{start_time};
|
||||
|
||||
$self->{test_output}->{$name} = "" unless($self->{verbose});
|
||||
|
||||
my $out = "";
|
||||
$out .= "[$self->{index}";
|
||||
if ($self->{totalsuites}) {
|
||||
$out .= "/$self->{totalsuites}";
|
||||
}
|
||||
$out.= " in ".$duration."s";
|
||||
$out .= sprintf(", %d errors", ($#{$self->{suitesfailed}}+1)) if ($#{$self->{suitesfailed}} > -1);
|
||||
$out .= "] $name";
|
||||
if ($self->{immediate}) {
|
||||
print "$out\n";
|
||||
} else {
|
||||
print "$out: ";
|
||||
}
|
||||
}
|
||||
|
||||
sub output_msg($$)
|
||||
{
|
||||
my ($self, $output) = @_;
|
||||
|
||||
if ($self->{verbose}) {
|
||||
require FileHandle;
|
||||
print $output;
|
||||
STDOUT->flush();
|
||||
} elsif (defined($self->{NAME})) {
|
||||
$self->{test_output}->{$self->{NAME}} .= $output;
|
||||
} else {
|
||||
print $output;
|
||||
}
|
||||
}
|
||||
|
||||
sub control_msg($$)
|
||||
{
|
||||
my ($self, $output) = @_;
|
||||
|
||||
#$self->output_msg($output);
|
||||
}
|
||||
|
||||
sub end_testsuite($$$$)
|
||||
{
|
||||
my ($self, $name, $result, $reason) = @_;
|
||||
my $out = "";
|
||||
my $unexpected = 0;
|
||||
|
||||
if (not defined($self->{test_output}->{$name})) {
|
||||
print "no output for name[$name]\n";
|
||||
}
|
||||
|
||||
if ($result eq "success" or $result eq "xfail") {
|
||||
$self->{suites_ok}++;
|
||||
} else {
|
||||
$self->output_msg("ERROR: Testsuite[$name]\nREASON: $reason\n");
|
||||
push (@{$self->{suitesfailed}}, $name);
|
||||
if ($self->{immediate} and not $self->{verbose}) {
|
||||
$out .= $self->{test_output}->{$name};
|
||||
}
|
||||
$unexpected = 1;
|
||||
}
|
||||
|
||||
if (not $self->{immediate}) {
|
||||
unless($unexpected) {
|
||||
$out .= " ok\n";
|
||||
} else {
|
||||
$out .= " " . uc($result) . "\n";
|
||||
}
|
||||
}
|
||||
|
||||
print $out;
|
||||
}
|
||||
|
||||
sub start_test($$$)
|
||||
{
|
||||
my ($self, $testname) = @_;
|
||||
}
|
||||
|
||||
sub end_test($$$$$)
|
||||
{
|
||||
my ($self, $testname, $result, $unexpected, $reason) = @_;
|
||||
|
||||
my $append = "";
|
||||
|
||||
unless ($unexpected) {
|
||||
$self->{test_output}->{$self->{NAME}} = "";
|
||||
if (not $self->{immediate}) {
|
||||
if ($result eq "failure") { print "f"; }
|
||||
elsif ($result eq "xfail") { print "X"; }
|
||||
elsif ($result eq "skip") { print "s"; }
|
||||
elsif ($result eq "success") { print "."; }
|
||||
else { print "?($result)"; }
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
chomp $reason;
|
||||
|
||||
$append = "UNEXPECTED($result): $testname\nREASON: $reason\n";
|
||||
|
||||
$self->{test_output}->{$self->{NAME}} .= $append;
|
||||
|
||||
if ($self->{immediate} and not $self->{verbose}) {
|
||||
print $self->{test_output}->{$self->{NAME}};
|
||||
$self->{test_output}->{$self->{NAME}} = "";
|
||||
}
|
||||
|
||||
if (not $self->{immediate}) {
|
||||
if ($result eq "error") { print "E"; }
|
||||
elsif ($result eq "failure") { print "F"; }
|
||||
elsif ($result eq "success") { print "S"; }
|
||||
else { print "?"; }
|
||||
}
|
||||
}
|
||||
|
||||
sub summary($)
|
||||
{
|
||||
my ($self) = @_;
|
||||
|
||||
open(SUMMARY, ">$self->{summaryfile}");
|
||||
|
||||
if ($#{$self->{suitesfailed}} > -1) {
|
||||
print SUMMARY "= Failed tests =\n";
|
||||
|
||||
foreach (@{$self->{suitesfailed}}) {
|
||||
print SUMMARY "== $_ ==\n";
|
||||
print SUMMARY $self->{test_output}->{$_}."\n\n";
|
||||
}
|
||||
|
||||
print SUMMARY "\n";
|
||||
}
|
||||
|
||||
if (not $self->{immediate} and not $self->{verbose}) {
|
||||
foreach (@{$self->{suitesfailed}}) {
|
||||
print "===============================================================================\n";
|
||||
print "FAIL: $_\n";
|
||||
print $self->{test_output}->{$_};
|
||||
print "\n";
|
||||
}
|
||||
}
|
||||
|
||||
print SUMMARY "= Skipped tests =\n";
|
||||
foreach my $reason (keys %{$self->{skips}}) {
|
||||
print SUMMARY "$reason\n";
|
||||
foreach my $name (@{$self->{skips}->{$reason}}) {
|
||||
print SUMMARY "\t$name\n";
|
||||
}
|
||||
print SUMMARY "\n";
|
||||
}
|
||||
close(SUMMARY);
|
||||
|
||||
print "\nA summary with detailed information can be found in:\n $self->{summaryfile}\n";
|
||||
|
||||
if ($#{$self->{suitesfailed}} == -1) {
|
||||
my $ok = $self->{statistics}->{TESTS_EXPECTED_OK} +
|
||||
$self->{statistics}->{TESTS_EXPECTED_FAIL};
|
||||
print "\nALL OK ($ok tests in $self->{suites_ok} testsuites)\n";
|
||||
} else {
|
||||
print "\nFAILED ($self->{statistics}->{TESTS_UNEXPECTED_FAIL} failures and $self->{statistics}->{TESTS_ERROR} errors in ". ($#{$self->{suitesfailed}}+1) ." testsuites)\n";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
sub skip_testsuite($$$)
|
||||
{
|
||||
my ($self, $name, $reason) = @_;
|
||||
|
||||
unless (defined($reason)) {
|
||||
$reason = "UNKNOWN";
|
||||
}
|
||||
push (@{$self->{skips}->{$reason}}, $name);
|
||||
|
||||
if ($self->{totalsuites}) {
|
||||
$self->{totalsuites}--;
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -28,24 +28,29 @@ def parse_results(msg_ops, statistics, fh):
|
||||
|
||||
while fh:
|
||||
l = fh.readline()
|
||||
if l == "":
|
||||
break
|
||||
if l.startswith("test: "):
|
||||
msg_ops.control_msg(l)
|
||||
name = l.split(":", 1)[1].strip()
|
||||
msg_ops.start_test(name)
|
||||
open_tests.append(name)
|
||||
elif l.startswith("time: "):
|
||||
(year, month, day, hour, minute, second) = re.match(
|
||||
"^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)\n/", l)
|
||||
msg_ops.report_time(time.mktime(second, minute, hour, day, month-1, year-1900))
|
||||
grp = re.match(
|
||||
"^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)\n", l)
|
||||
msg_ops.report_time(time.mktime((int(grp.group(1)), int(grp.group(2)), int(grp.group(3)), int(grp.group(4)), int(grp.group(5)), int(grp.group(6)), 0, 0, 0)))
|
||||
elif re.match("^(" + "|".join(VALID_RESULTS) + "): (.*?)( \[)?([ \t]*)( multipart)?\n", l):
|
||||
msg_ops.control_msg(l)
|
||||
(result, testname, hasreason) = re.match("^(" + "|".join(VALID_RESULTS) + "): (.*?)( \[)?([ \t]*)( multipart)?\n", l)
|
||||
grp = re.match("^(" + "|".join(VALID_RESULTS) + "): (.*?)( \[)?([ \t]*)( multipart)?\n", l)
|
||||
(result, testname, hasreason) = (grp.group(1), grp.group(2), grp.group(3))
|
||||
if hasreason:
|
||||
reason = ""
|
||||
# reason may be specified in next lines
|
||||
terminated = False
|
||||
while fh:
|
||||
l = fh.readline()
|
||||
if l == "":
|
||||
break
|
||||
msg_ops.control_msg(l)
|
||||
if l == "]\n":
|
||||
terminated = True
|
||||
@ -58,6 +63,8 @@ def parse_results(msg_ops, statistics, fh):
|
||||
msg_ops.end_test(testname, "error", 1,
|
||||
"reason (%s) interrupted" % result)
|
||||
return 1
|
||||
else:
|
||||
reason = None
|
||||
if result in ("success", "successful"):
|
||||
open_tests.pop() #FIXME: Check that popped value == $testname
|
||||
statistics['TESTS_EXPECTED_OK']+=1
|
||||
@ -104,12 +111,6 @@ def parse_results(msg_ops, statistics, fh):
|
||||
"was started but never finished!")
|
||||
statistics['TESTS_ERROR']+=1
|
||||
|
||||
# if the Filter module is in use, it will have the right counts
|
||||
if 'total_error' in msg_ops:
|
||||
statistics['TESTS_ERROR'] = msg_ops['total_error']
|
||||
statistics['TESTS_UNEXPECTED_FAIL'] = msg_ops['total_fail']
|
||||
statistics['TESTS_EXPECTED_FAIL'] = msg_ops['total_xfail']
|
||||
|
||||
if statistics['TESTS_ERROR'] > 0:
|
||||
return 1
|
||||
if statistics['TESTS_UNEXPECTED_FAIL'] > 0:
|
||||
|
||||
@ -3253,7 +3253,7 @@ selftest:: all torture timelimit
|
||||
--socket-wrapper $(TESTS) | \
|
||||
$(PERL) $(selftestdir)/filter-subunit.pl \
|
||||
--expected-failures=$(srcdir)/selftest/knownfail | \
|
||||
$(PERL) $(selftestdir)/format-subunit --immediate
|
||||
$(PYTHON) $(selftestdir)/format-subunit --immediate
|
||||
|
||||
selftest-%:
|
||||
$(MAKE) selftest TESTS=$*
|
||||
|
||||
@ -14,7 +14,7 @@ ST_DONE_TEST = @test -f $(selftest_prefix)/st_done || { echo "SELFTEST FAILED";
|
||||
SELFTEST_NOSLOW_OPTS = --exclude=$(srcdir)/selftest/slow
|
||||
SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selftest/quick
|
||||
FILTER_XFAIL = $(PERL) $(selftestdir)/filter-subunit.pl --expected-failures=$(srcdir)/selftest/knownfail
|
||||
SUBUNIT_FORMATTER ?= $(PERL) $(selftestdir)/format-subunit --prefix=${selftest_prefix} --immediate
|
||||
SUBUNIT_FORMATTER ?= $(PYTHON) $(selftestdir)/format-subunit --prefix=${selftest_prefix} --immediate
|
||||
FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(SUBUNIT_FORMATTER)
|
||||
|
||||
test-subunit:: everything
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user