mirror of
https://github.com/samba-team/samba.git
synced 2024-12-22 13:34:15 +03:00
selftest: Use filter-xfail for known failures
This commit is contained in:
Jelmer Vernooij
parent
4e276fb60a
commit
24f01e70aa
@ -23,9 +23,9 @@ require Exporter;
|
||||
|
||||
use strict;
|
||||
|
||||
sub parse_results($$$$$)
|
||||
sub parse_results($$$$)
|
||||
{
|
||||
my ($msg_ops, $statistics, $fh, $expecting_failure, $open_tests) = @_;
|
||||
my ($msg_ops, $statistics, $fh, $open_tests) = @_;
|
||||
my $unexpected_ok = 0;
|
||||
my $expected_fail = 0;
|
||||
my $unexpected_fail = 0;
|
||||
@ -39,7 +39,7 @@ sub parse_results($$$$$)
|
||||
push (@$open_tests, $1);
|
||||
} elsif (/^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)Z\n/) {
|
||||
$msg_ops->report_time(mktime($6, $5, $4, $3, $2, $1));
|
||||
} elsif (/^(success|successful|failure|fail|skip|knownfail|error): (.*?)( \[)?([ \t]*)\n/) {
|
||||
} elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail): (.*?)( \[)?([ \t]*)\n/) {
|
||||
$msg_ops->control_msg($_);
|
||||
my $reason = undef;
|
||||
if ($3) {
|
||||
@ -60,29 +60,18 @@ sub parse_results($$$$$)
|
||||
my $result = $1;
|
||||
if ($1 eq "success" or $1 eq "successful") {
|
||||
pop(@$open_tests); #FIXME: Check that popped value == $2
|
||||
if ($expecting_failure->(join(".", @$open_tests) . ".$2")) {
|
||||
$statistics->{TESTS_UNEXPECTED_OK}++;
|
||||
$msg_ops->end_test($open_tests, $2, $1, 1, $reason);
|
||||
$unexpected_ok++;
|
||||
} else {
|
||||
$statistics->{TESTS_EXPECTED_OK}++;
|
||||
$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
|
||||
}
|
||||
} elsif ($1 eq "failure" or $1 eq "fail") {
|
||||
pop(@$open_tests); #FIXME: Check that popped value == $2
|
||||
if ($expecting_failure->(join(".", @$open_tests) . ".$2")) {
|
||||
$statistics->{TESTS_EXPECTED_FAIL}++;
|
||||
$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
|
||||
$expected_fail++;
|
||||
} else {
|
||||
$statistics->{TESTS_UNEXPECTED_FAIL}++;
|
||||
$msg_ops->end_test($open_tests, $2, $1, 1, $reason);
|
||||
$unexpected_fail++;
|
||||
}
|
||||
} elsif ($1 eq "knownfail") {
|
||||
$statistics->{TESTS_EXPECTED_OK}++;
|
||||
$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
|
||||
} elsif ($1 eq "xfail" or $1 eq "knownfail") {
|
||||
pop(@$open_tests); #FIXME: Check that popped value == $2
|
||||
$statistics->{TESTS_EXPECTED_FAIL}++;
|
||||
$msg_ops->end_test($open_tests, $2, $1, 0, $reason);
|
||||
$expected_fail++;
|
||||
} elsif ($1 eq "failure" or $1 eq "fail") {
|
||||
pop(@$open_tests); #FIXME: Check that popped value == $2
|
||||
$statistics->{TESTS_UNEXPECTED_FAIL}++;
|
||||
$msg_ops->end_test($open_tests, $2, $1, 1, $reason);
|
||||
$unexpected_fail++;
|
||||
} elsif ($1 eq "skip") {
|
||||
$statistics->{TESTS_SKIP}++;
|
||||
pop(@$open_tests); #FIXME: Check that popped value == $2
|
||||
|
||||
@ -157,6 +157,6 @@ sub end_test($$$$$)
|
||||
my $msg_ops = {};
|
||||
bless $msg_ops;
|
||||
|
||||
parse_results($msg_ops, $statistics, *STDIN, sub { return 0; }, []);
|
||||
parse_results($msg_ops, $statistics, *STDIN, []);
|
||||
|
||||
0;
|
||||
|
||||
@ -61,7 +61,7 @@ if ($opt_format eq "buildfarm") {
|
||||
}
|
||||
|
||||
my $expected_ret = parse_results(
|
||||
$msg_ops, $statistics, *STDIN, sub { return 0; }, []);
|
||||
$msg_ops, $statistics, *STDIN, []);
|
||||
|
||||
$msg_ops->summary();
|
||||
|
||||
|
||||
@ -77,23 +77,10 @@ when the server is running locally.
|
||||
Will prevent TCP and UDP ports being opened on the local host but
|
||||
(transparently) redirects these calls to use unix domain sockets.
|
||||
|
||||
=item I<--expected-failures>
|
||||
|
||||
Specify a file containing a list of tests that are expected to fail. Failures for
|
||||
these tests will be counted as successes, successes will be counted as failures.
|
||||
|
||||
The format for the file is, one entry per line:
|
||||
|
||||
TESTSUITE-NAME.TEST-NAME
|
||||
|
||||
The reason for a test can also be specified, by adding a hash sign (#) and the reason
|
||||
after the test name.
|
||||
|
||||
=item I<--exclude>
|
||||
|
||||
Specify a file containing a list of tests that should be skipped. Possible
|
||||
candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
|
||||
for the --expected-failures flag.
|
||||
candidates are tests that segfault the server, flip or don't end.
|
||||
|
||||
=item I<--include>
|
||||
|
||||
@ -157,7 +144,6 @@ my $opt_socket_wrapper_pcap = undef;
|
||||
my $opt_socket_wrapper_keep_pcap = undef;
|
||||
my $opt_one = 0;
|
||||
my $opt_immediate = 0;
|
||||
my $opt_expected_failures = undef;
|
||||
my @opt_exclude = ();
|
||||
my @opt_include = ();
|
||||
my $opt_verbose = 0;
|
||||
@ -176,7 +162,6 @@ my $builddir = ".";
|
||||
my $exeext = "";
|
||||
my $prefix = "./st";
|
||||
|
||||
my @expected_failures = ();
|
||||
my @includes = ();
|
||||
my @excludes = ();
|
||||
|
||||
@ -205,12 +190,6 @@ sub find_in_list($$)
|
||||
return undef;
|
||||
}
|
||||
|
||||
sub expecting_failure($)
|
||||
{
|
||||
my ($name) = @_;
|
||||
return find_in_list(\@expected_failures, $name);
|
||||
}
|
||||
|
||||
sub skip($)
|
||||
{
|
||||
my ($name) = @_;
|
||||
@ -265,7 +244,7 @@ sub run_testsuite($$$$$$)
|
||||
}
|
||||
|
||||
my $expected_ret = parse_results(
|
||||
$msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
|
||||
$msg_ops, $statistics, *RESULT, [$name]);
|
||||
|
||||
my $envlog = getlog_env($envname);
|
||||
$msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
|
||||
@ -323,7 +302,6 @@ Target Specific:
|
||||
failed
|
||||
--socket-wrapper enable socket wrapper
|
||||
--bindir=PATH path to target binaries
|
||||
--expected-failures=FILE specify list of tests that is guaranteed to fail
|
||||
|
||||
Samba4 Specific:
|
||||
--ldap=openldap|fedora-ds back samba onto specified ldap server
|
||||
@ -351,7 +329,6 @@ my $result = GetOptions (
|
||||
'quick' => \$opt_quick,
|
||||
'one' => \$opt_one,
|
||||
'immediate' => \$opt_immediate,
|
||||
'expected-failures=s' => \$opt_expected_failures,
|
||||
'exclude=s' => \@opt_exclude,
|
||||
'include=s' => \@opt_include,
|
||||
'srcdir=s' => \$srcdir,
|
||||
@ -537,10 +514,6 @@ sub read_test_regexes($)
|
||||
return @ret;
|
||||
}
|
||||
|
||||
if (defined($opt_expected_failures)) {
|
||||
@expected_failures = read_test_regexes($opt_expected_failures);
|
||||
}
|
||||
|
||||
foreach (@opt_exclude) {
|
||||
push (@excludes, read_test_regexes($_));
|
||||
}
|
||||
|
||||
@ -3,14 +3,17 @@ TEST_FORMAT = plain
|
||||
SELFTEST = $(LD_LIBPATH_OVERRIDE) PYTHON=$(PYTHON) \
|
||||
$(PERL) $(selftestdir)/selftest.pl --prefix=${selftest_prefix} \
|
||||
--builddir=$(builddir) --srcdir=$(srcdir) \
|
||||
--expected-failures=$(srcdir)/selftest/knownfail \
|
||||
--format=subunit \
|
||||
--exclude=$(srcdir)/selftest/skip --testlist="./selftest/tests.sh|" \
|
||||
$(TEST_OPTIONS)
|
||||
|
||||
SELFTEST_NOSLOW_OPTS = --exclude=$(srcdir)/selftest/slow
|
||||
SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selftest/quick
|
||||
FORMAT_TEST_OUTPUT = $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
|
||||
FILTER_XFAIL = $(PERL) $(selftestdir)/filter-xfail.pl --expected-failures=$(srcdir)/selftest/knownfail
|
||||
FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
|
||||
|
||||
subunittest:: everything
|
||||
$(SELFTEST) $(TESTS)
|
||||
|
||||
slowtest:: everything
|
||||
$(SELFTEST) $(DEFAULT_TEST_OPTIONS) --immediate $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate
|
||||
|
||||
Loading…
Reference in New Issue
Block a user