1
0
mirror of https://github.com/samba-team/samba.git synced 2025-03-27 22:50:26 +03:00

selftest: Use external processes for filtering known failures and pretty

formatting.
This commit is contained in:
Jelmer Vernooij 2009-06-04 13:49:11 +02:00
parent 9bdf76c9c2
commit 16a69f358a
8 changed files with 128 additions and 131 deletions

View File

@ -41,6 +41,8 @@ sub parse_results($$$$)
$msg_ops->report_time(mktime($6, $5, $4, $3, $2, $1));
} elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail): (.*?)( \[)?([ \t]*)\n/) {
$msg_ops->control_msg($_);
my $result = $1;
my $testname = $2;
my $reason = undef;
if ($3) {
$reason = "";
@ -53,33 +55,35 @@ sub parse_results($$$$)
unless ($terminated) {
$statistics->{TESTS_ERROR}++;
$msg_ops->end_test($open_tests, $2, $1, 1, "reason interrupted");
$msg_ops->end_test($open_tests, $testname, $result, 1, "reason interrupted");
return 1;
}
}
my $result = $1;
if ($1 eq "success" or $1 eq "successful") {
pop(@$open_tests); #FIXME: Check that popped value == $2
if ($result eq "success" or $result eq "successful") {
pop(@$open_tests); #FIXME: Check that popped value == $testname
$statistics->{TESTS_EXPECTED_OK}++;
$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
} elsif ($1 eq "xfail" or $1 eq "knownfail") {
pop(@$open_tests); #FIXME: Check that popped value == $2
$msg_ops->end_test($open_tests, $testname, $result, 0, $reason);
} elsif ($result eq "xfail" or $result eq "knownfail") {
pop(@$open_tests); #FIXME: Check that popped value == $testname
$statistics->{TESTS_EXPECTED_FAIL}++;
$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
$msg_ops->end_test($open_tests, $testname, $result, 0, $reason);
$expected_fail++;
} elsif ($1 eq "failure" or $1 eq "fail") {
pop(@$open_tests); #FIXME: Check that popped value == $2
} elsif ($result eq "failure" or $result eq "fail") {
pop(@$open_tests); #FIXME: Check that popped value == $testname
$statistics->{TESTS_UNEXPECTED_FAIL}++;
$msg_ops->end_test($open_tests, $2, $1, 1, $reason);
$msg_ops->end_test($open_tests, $testname, $result, 1, $reason);
$unexpected_fail++;
} elsif ($1 eq "skip") {
} elsif ($result eq "skip") {
$statistics->{TESTS_SKIP}++;
pop(@$open_tests); #FIXME: Check that popped value == $2
$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
} elsif ($1 eq "error") {
my $last = pop(@$open_tests);
if (defined($last) and $last ne $testname) {
push (@$open_tests, $testname);
}
$msg_ops->end_test($open_tests, $testname, $result, 0, $reason);
} elsif ($result eq "error") {
$statistics->{TESTS_ERROR}++;
pop(@$open_tests); #FIXME: Check that popped value == $2
$msg_ops->end_test($open_tests, $2, $1, 1, $reason);
pop(@$open_tests); #FIXME: Check that popped value == $testname
$msg_ops->end_test($open_tests, $testname, $result, 1, $reason);
$unexpected_err++;
}
} else {
@ -114,7 +118,9 @@ sub end_test($$;$)
my $result = shift;
my $reason = shift;
if ($reason) {
print "$result: $name [ $reason ]\n";
print "$result: $name [";
print "$reason";
print "]\n";
} else {
print "$result: $name\n";
}

View File

@ -56,13 +56,20 @@ use lib "$RealBin";
use Subunit qw(parse_results);
my $opt_expected_failures = undef;
my $opt_help = 0;
my @expected_failures = ();
my $result = GetOptions(
'expected-failures=s' => \$opt_expected_failures,
'help' => \$opt_help,
);
exit(1) if (not $result);
if ($opt_help) {
print "Usage: filter-xfail [--expected-failures=FILE]... < instream > outstream\n";
exit(0);
}
sub read_test_regexes($)
{
my ($name) = @_;

View File

@ -3,6 +3,38 @@
# Copyright (C) Jelmer Vernooij <jelmer@samba.org>
# Published under the GNU GPL, v3 or later
=pod
=head1 NAME
format-subunit [--format=<NAME>] [--immediate] < instream > outstream
=head1 SYNOPSIS
Format the output of a subunit stream.
=head1 OPTIONS
=over 4
=item I<--immediate>
Show errors as soon as they happen rather than at the end of the test run.
=item I<--format>=FORMAT
Choose the format to print. Currently supported are plain, html or buildfarm.
=head1 LICENSE
GNU General Public License, version 3 or later.
=head1 AUTHOR
Jelmer Vernooij <jelmer@samba.org>
=cut
use Getopt::Long;
use strict;
use FindBin qw($RealBin $Script);
@ -60,8 +92,7 @@ if ($opt_format eq "buildfarm") {
die("Invalid output format '$opt_format'");
}
my $expected_ret = parse_results(
$msg_ops, $statistics, *STDIN, []);
my $expected_ret = parse_results($msg_ops, $statistics, *STDIN, []);
$msg_ops->summary();

View File

@ -29,9 +29,10 @@ use BuildFarm;
use strict;
sub new($$$) {
my ($class) = @_;
my ($class, $statistics) = @_;
my $self = {
test_output => {},
statistics => $statistics,
last_time => 0,
start_time => undef,
};
@ -111,6 +112,8 @@ sub summary($)
my ($self) = @_;
BuildFarm::summary($self->{last_time} - $self->{start_time});
print "TEST STATUS: $self->{statistics}->{SUITES_FAIL}\n";
}
sub skip_testsuite($$$)

View File

@ -87,8 +87,10 @@ sub output_msg($$)
require FileHandle;
print $output;
STDOUT->flush();
} else {
} elsif (defined($self->{NAME})) {
$self->{test_output}->{$self->{NAME}} .= $output;
} else {
print $output;
}
}
@ -96,7 +98,7 @@ sub control_msg($$)
{
my ($self, $output) = @_;
$self->output_msg($output);
#$self->output_msg($output);
}
sub end_testsuite($$$$$)
@ -157,6 +159,7 @@ sub end_test($$$$$)
$self->{test_output}->{$self->{NAME}} = "";
if (not $self->{immediate}) {
if ($result eq "failure") { print "f"; }
elsif ($result eq "xfail") { print "X"; }
elsif ($result eq "skip") { print "s"; }
elsif ($result eq "success") { print "."; }
else { print "?($result)"; }
@ -231,10 +234,13 @@ sub summary($)
}
sub skip_testsuite($$)
sub skip_testsuite($$$)
{
my ($self, $name, $reason) = @_;
unless (defined($reason)) {
$reason = "UNKNOWN";
}
push (@{$self->{skips}->{$reason}}, $name);
if ($self->{totalsuites}) {

View File

@ -95,7 +95,7 @@ sub skip_testsuite($$$$)
my ($self, $name, $reason) = @_;
Subunit::start_test($name);
Subunit::end_test($name, "skip");
Subunit::end_test($name, "skip", $reason);
}
1;

View File

@ -26,7 +26,7 @@ selftest - Samba test runner
selftest --help
selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
=head1 DESCRIPTION
@ -56,10 +56,6 @@ Executable extention
Change directory to run tests in. Default is 'st'.
=item I<--immediate>
Show errors as soon as they happen rather than at the end of the test run.
=item I<--target samba4|samba3|win|kvm>
Specify test target against which to run. Default is 'samba4'.
@ -143,7 +139,6 @@ my $opt_socket_wrapper = 0;
my $opt_socket_wrapper_pcap = undef;
my $opt_socket_wrapper_keep_pcap = undef;
my $opt_one = 0;
my $opt_immediate = 0;
my @opt_exclude = ();
my @opt_include = ();
my $opt_verbose = 0;
@ -154,7 +149,6 @@ my $opt_analyse_cmd = undef;
my $opt_resetup_env = undef;
my $opt_bindir = undef;
my $opt_no_lazy_setup = undef;
my $opt_format = "plain";
my @testlists = ();
my $srcdir = ".";
@ -165,17 +159,6 @@ my $prefix = "./st";
my @includes = ();
my @excludes = ();
my $statistics = {
SUITES_FAIL => 0,
TESTS_UNEXPECTED_OK => 0,
TESTS_EXPECTED_OK => 0,
TESTS_UNEXPECTED_FAIL => 0,
TESTS_EXPECTED_FAIL => 0,
TESTS_ERROR => 0,
TESTS_SKIP => 0,
};
sub find_in_list($$)
{
my ($list, $fullname) = @_;
@ -216,65 +199,61 @@ sub setup_pcap($)
return $pcap_file;
}
sub cleanup_pcap($$$)
sub cleanup_pcap($$)
{
my ($pcap_file, $expected_ret, $ret) = @_;
my ($pcap_file, $exitcode) = @_;
return unless ($opt_socket_wrapper_pcap);
return if ($opt_socket_wrapper_keep_pcap);
return unless ($expected_ret == $ret);
return unless ($exitcode == 0);
return unless defined($pcap_file);
unlink($pcap_file);
}
sub run_testsuite($$$$$$)
sub run_testsuite($$$$$)
{
my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
my ($envname, $name, $cmd, $i, $totalsuites) = @_;
my $pcap_file = setup_pcap($name);
$msg_ops->report_time(time());
$msg_ops->start_test([], $name);
Subunit::report_time(time());
Subunit::start_test($name);
unless (open(RESULT, "$cmd 2>&1|")) {
$statistics->{TESTS_ERROR}++;
$msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
$statistics->{SUITES_FAIL}++;
my $ret = system("$cmd 2>&1");
if ($ret == -1) {
Subunit::end_test($name, "error", "Unable to run $cmd: $!");
return 0;
}
my $expected_ret = parse_results(
$msg_ops, $statistics, *RESULT, [$name]);
my $envlog = getlog_env($envname);
$msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
$msg_ops->output_msg("CMD: $cmd\n");
my $ret = close(RESULT);
$ret = 0 unless $ret == 1;
my $exitcode = $? >> 8;
$msg_ops->report_time(time());
if ($ret == 1) {
$msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
} else {
$msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
if ($envlog ne "") {
print "ENVLOG: $envlog\n";
}
cleanup_pcap($pcap_file, $expected_ret, $ret);
print "CMD: $cmd\n";
my $exitcode = $ret >> 8;
Subunit::report_time(time());
my $reason = "Exit code was $exitcode";
my $result;
if ($exitcode == 0) {
$result = "success";
} else {
$result = "failure";
}
Subunit::end_test($name, $result, $reason);
cleanup_pcap($pcap_file, $exitcode);
if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
$msg_ops->output_msg("PCAP FILE: $pcap_file\n");
print "PCAP FILE: $pcap_file\n";
}
if ($ret != $expected_ret) {
$statistics->{SUITES_FAIL}++;
if ($exitcode != 0) {
exit(1) if ($opt_one);
}
return ($ret == $expected_ret);
return $exitcode;
}
sub ShowHelp()
@ -312,7 +291,6 @@ Kvm Specific:
Behaviour:
--quick run quick overall test
--one abort when the first test fails
--immediate print test output for failed tests during run
--verbose be verbose
--analyse-cmd CMD command to run after each test
";
@ -328,7 +306,6 @@ my $result = GetOptions (
'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
'quick' => \$opt_quick,
'one' => \$opt_one,
'immediate' => \$opt_immediate,
'exclude=s' => \@opt_exclude,
'include=s' => \@opt_include,
'srcdir=s' => \$srcdir,
@ -341,7 +318,6 @@ my $result = GetOptions (
'no-lazy-setup' => \$opt_no_lazy_setup,
'resetup-environment' => \$opt_resetup_env,
'bindir:s' => \$opt_bindir,
'format=s' => \$opt_format,
'image=s' => \$opt_image,
'testlist=s' => \@testlists
);
@ -401,11 +377,6 @@ $ENV{BUILDDIR} = $builddir;
$ENV{BUILDDIR_ABS} = $builddir_abs;
$ENV{EXEEXT} = $exeext;
if (defined($ENV{RUN_FROM_BUILD_FARM}) and
($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
$opt_format = "buildfarm";
}
my $tls_enabled = not $opt_quick;
$ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
$ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
@ -650,30 +621,13 @@ foreach my $fn (@testlists) {
}
}
my $msg_ops;
if ($opt_format eq "buildfarm") {
require output::buildfarm;
$msg_ops = new output::buildfarm($statistics);
} elsif ($opt_format eq "plain") {
require output::plain;
$msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
} elsif ($opt_format eq "html") {
require output::html;
mkdir("test-results", 0777);
$msg_ops = new output::html("test-results", $statistics);
} elsif ($opt_format eq "subunit") {
require output::subunit;
$msg_ops = new output::subunit();
} else {
die("Invalid output format '$opt_format'");
}
$msg_ops->report_time(time());
Subunit::report_time(time());
foreach (@available) {
my $name = $$_[0];
my $skipreason = skip($name);
if ($skipreason) {
$msg_ops->skip_testsuite($name, $skipreason);
Subunit::end_test($name, "skip", $skipreason);
} else {
push(@todo, $_);
}
@ -861,13 +815,12 @@ $envvarstr
my $envvars = setup_env($envname);
if (not defined($envvars)) {
$msg_ops->skip_testsuite($name,
Subunit::end_test($name, "skip",
"unable to set up environment $envname");
next;
}
run_testsuite($envname, $name, $cmd, $i, $suitestotal,
$msg_ops);
run_testsuite($envname, $name, $cmd, $i, $suitestotal);
if (defined($opt_analyse_cmd)) {
system("$opt_analyse_cmd \"$name\"");
@ -883,8 +836,6 @@ teardown_env($_) foreach (keys %running_envs);
$target->stop();
$msg_ops->summary();
my $failed = 0;
# if there were any valgrind failures, show them
@ -897,9 +848,4 @@ foreach (<$prefix/valgrind.log*>) {
system("cat $_");
}
}
if ($opt_format eq "buildfarm") {
print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
}
exit $statistics->{SUITES_FAIL};
exit 0;

View File

@ -3,7 +3,6 @@ TEST_FORMAT = plain
SELFTEST = $(LD_LIBPATH_OVERRIDE) PYTHON=$(PYTHON) \
$(PERL) $(selftestdir)/selftest.pl --prefix=${selftest_prefix} \
--builddir=$(builddir) --srcdir=$(srcdir) \
--format=subunit \
--exclude=$(srcdir)/selftest/skip --testlist="./selftest/tests.sh|" \
$(TEST_OPTIONS)
@ -13,40 +12,39 @@ FILTER_XFAIL = $(PERL) $(selftestdir)/filter-xfail.pl --expected-failures=$(srcd
FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
subunittest:: everything
$(SELFTEST) $(TESTS)
$(SELFTEST) --socket-wrapper $(TESTS)
slowtest:: everything
$(SELFTEST) $(DEFAULT_TEST_OPTIONS) --immediate $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
$(SELFTEST) $(DEFAULT_TEST_OPTIONS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
test:: everything
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) --immediate \
$(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
kvmtest:: everything
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) --immediate \
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) \
--target=kvm --image=$(KVM_IMAGE) | $(FORMAT_TEST_OUTPUT) --immediate
kvmquicktest:: everything
$(SELFTEST) $(DEFAULT_TEST_OPTIONS) --immediate \
$(SELFTEST) $(DEFAULT_TEST_OPTIONS) \
$(SELFTEST_QUICK_OPTS) --target=kvm --image=$(KVM_IMAGE) | $(FORMAT_TEST_OUTPUT) | $(FORMAT_TEST_OUTPUT) --immediate
testone:: everything
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(DEFAULT_TEST_OPTIONS) --one $(TESTS) | $(FORMAT_TEST_OUTPUT)
test-swrap:: everything
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper --immediate $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
test-swrap-pcap:: everything
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-pcap --immediate $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-pcap $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
test-swrap-keep-pcap:: everything
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-keep-pcap --immediate $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper-keep-pcap $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
test-noswrap:: everything
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --immediate $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
quicktest:: all
$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper --immediate $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
quicktestone:: all
$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper --one $(TESTS) | $(FORMAT_TEST_OUTPUT)
@ -65,12 +63,12 @@ valgrindtest:: valgrindtest-all
valgrindtest-quick:: all
SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
$(SELFTEST) $(SELFTEST_QUICK_OPTS) --immediate --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
valgrindtest-all:: everything
SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
VALGRIND="valgrind -q --num-callers=30 --log-file=${selftest_prefix}/valgrind.log" \
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --immediate --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
valgrindtest-env:: everything
SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/valgrind_run $(LD_LIBPATH_OVERRIDE)" \
@ -81,11 +79,11 @@ gdbtest:: gdbtest-all
gdbtest-quick:: all
SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \
$(SELFTEST) $(SELFTEST_QUICK_OPTS) --immediate --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
$(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
gdbtest-all:: everything
SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --immediate --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
$(SELFTEST) $(SELFTEST_NOSLOW_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
gdbtest-env:: everything
SAMBA_VALGRIND="xterm -n server -e $(selftestdir)/gdb_run $(LD_LIBPATH_OVERRIDE)" \