r21707: Finally merge my (long-living) perlselftest branch.

This changes the main selftest code to be in perl rather than in shell script.

The selftest script is now no longer a black box but a regular executable that takes
--help.

This adds the following features:

 * "make test TESTS=foo" will run only the tests that match the regex "foo"
 * ability to deal with expected failures. the suite will not warn about tests
   that fail and are known to fail, but will warn about other failing tests and
   tests that are succeeding tests but incorrectly marked as failing.
 * ability to print a summary with all failures at the end of the run

It also opens up the way to the following features, which I hope to implement later:
 * "environments", for example having a complete domains with DCs and domain members
 in a testenvironment
 * only set up smbd if necessary (not when running LOCAL tests, for example)
 * different mktestsetup scripts per target. except for the mktestsetup script, we can
   use the same infrastructure for samba 3 or windows.

source/script/tests/Samba4.pm

@@ -0,0 +1,139 @@
+#!/usr/bin/perl
+# Bootstrap Samba and run a number of tests against it.
+# Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
+# Published under the GNU GPL, v3 or later.
+
+package Samba4;
+
+use Exporter;
+@ISA = qw(Exporter);
+@EXPORT_OK = qw(slapd_start slapd_stop smbd_check_or_start provision);
+
+use strict;
+use FindBin qw($RealBin);
+use POSIX;
+
+sub slapd_start($$)
+{
+	my ($conf, $uri) = @_;
+	if (defined($ENV{FEDORA_DS_PREFIX})) {
+		system("$ENV{FEDORA_DS_PREFIX}/lib/fedora-ds/ds_newinst.pl $ENV{FEDORA_DS_INF}") or die("Unable to provision fedora ds ldapd");
+	} else {
+		my $oldpath = $ENV{PATH};
+		$ENV{PATH} = "/usr/local/sbin:/usr/sbin:/sbin:$ENV{PATH}";
+		# running slapd in the background means it stays in the same process group, so it can be
+		# killed by timelimit
+		system("slapd -d0 -f $conf -h $uri &");
+		$ENV{PATH} = $oldpath;
+	}
+    return $? >> 8;
+}
+
+sub slapd_stop()
+{
+	if (defined($ENV{FEDORA_DS_PREFIX})) {
+		system("$ENV{LDAPDIR}/slapd-samba4/stop-slapd");
+	} else {
+		open(IN, "<$ENV{PIDDIR}/slapd.pid") or 
+			die("unable to open slapd pid file");
+		kill 9, <IN>;
+		close(IN);
+	}
+}
+
+sub smbd_check_or_start($$$$$$) 
+{
+	my ($bindir, $test_fifo, $test_log, $socket_wrapper_dir, $max_time, $conffile) = @_;
+	return 0 if ( -p $test_fifo );
+
+	warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless
+		defined($socket_wrapper_dir) or $< == 0;
+
+	if (defined($socket_wrapper_dir)) {
+		if ( -d $socket_wrapper_dir ) {
+			unlink <$socket_wrapper_dir/*>;
+		} else {
+			mkdir($socket_wrapper_dir);
+		}
+	}
+
+	unlink($test_fifo);
+	POSIX::mkfifo($test_fifo, 0700);
+	unlink($test_log);
+	
+	my $valgrind = "";
+	if (defined($ENV{SMBD_VALGRIND})) {
+		$valgrind = $ENV{SMBD_VALGRIND};
+	} 
+
+	print "STARTING SMBD...";
+	my $pid = fork();
+	if ($pid == 0) {
+		open STDIN, $test_fifo;
+		open STDOUT, ">$test_log";
+		open STDERR, '>&STDOUT';
+		my $optarg = "";
+		if (defined($max_time)) {
+			$optarg = "--maximum-runtime=$max_time ";
+		}
+		my $ret = system("$valgrind $bindir/smbd $optarg -s $conffile -M single -i --leak-report-full");
+		if ($? == -1) {
+			print "Unable to start smbd: $ret: $!\n";
+			exit 1;
+		}
+		unlink($test_fifo);
+		unlink(<$socket_wrapper_dir/*>) if (defined($socket_wrapper_dir) and -d $socket_wrapper_dir);
+		my $exit = $? >> 8;
+		if ( $ret == 0 ) {
+			print "smbd exits with status $exit\n";
+		} elsif ( $ret & 127 ) {
+			print "smbd got signal ".($ret & 127)." and exits with $exit!\n";
+		} else {
+			$ret = $? >> 8;
+			print "smbd failed with status $exit!\n";
+		}
+		exit $exit;
+	}
+	print "DONE\n";
+
+	return $pid;
+}
+
+sub wait_for_start()
+{
+	# give time for nbt server to register its names
+	print "delaying for nbt name registration\n";
+	sleep(4);
+
+	# This will return quickly when things are up, but be slow if we 
+	# need to wait for (eg) SSL init 
+	system("bin/nmblookup $ENV{CONFIGURATION} $ENV{SERVER}");
+	system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{SERVER}");
+	system("bin/nmblookup $ENV{CONFIGURATION} $ENV{SERVER}");
+	system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{NETBIOSNAME}");
+	system("bin/nmblookup $ENV{CONFIGURATION} $ENV{NETBIOSNAME}");
+	system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{NETBIOSNAME}");
+}
+
+sub provision($)
+{
+	my ($prefix) = @_;
+	my %ret = ();
+	print "PROVISIONING...";
+	open(IN, "$RealBin/mktestsetup.sh $prefix|") or die("Unable to setup");
+	while (<IN>) {
+		die ("Error parsing `$_'") unless (/^([A-Z0-9a-z_]+)=(.*)$/);
+		$ret{$1} = $2;
+	}
+	close(IN);
+	return \%ret;
+}
+
+sub provision_ldap($$)
+{
+	my ($bindir, $setupdir) = @_;
+    system("$bindir/smbscript $setupdir/provision $ENV{PROVISION_OPTIONS} \"$ENV{PROVISION_ACI}\" --ldap-backend=$ENV{LDAPI}") or
+		die("LDAP PROVISIONING failed: $bindir/smbscript $setupdir/provision $ENV{PROVISION_OPTIONS} \"$ENV{PROVISION_ACI}\" --ldap-backend=$ENV{LDAPI}");
+}
+
+1;

source/script/tests/SocketWrapper.pm

@@ -0,0 +1,34 @@
+#!/usr/bin/perl
+# Bootstrap Samba and run a number of tests against it.
+# Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
+# Published under the GNU GPL, v3 or later.
+
+package SocketWrapper;
+
+use Exporter;
+@ISA = qw(Exporter);
+@EXPORT_OK = qw(setup_dir setup_pcap set_default_iface);
+
+use strict;
+use FindBin qw($RealBin);
+
+sub setup_dir($)
+{
+	my ($dir) = @_;
+	$ENV{SOCKET_WRAPPER_DIR} = $dir;
+	return $dir;
+}
+
+sub setup_pcap($)
+{
+	my ($pcap_file) = @_;
+
+}
+
+sub set_default_iface($)
+{
+	my ($i) = @_;
+	$ENV{SOCKET_WRAPPER_DEFAULT_IFACE} = $i;
+}
+
+1;

source/script/tests/selftest.pl

@@ -2,6 +2,107 @@
 # Bootstrap Samba and run a number of tests against it.
 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
 # Published under the GNU GPL, v3 or later.
+
+=pod
+
+=head1 NAME
+
+selftest - Samba test runner
+
+=head1 SYNOPSIS
+
+selftest --help
+
+selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
+
+=head1 DESCRIPTION
+
+A simple test runner. TESTS is a regular expression with tests to run.
+
+=head1 OPTIONS
+
+=over 4
+
+=item I<--help>
+
+Show list of available options.
+
+=item I<--srcdir=DIR>
+
+Source directory.
+
+=item I<--builddir=DIR>
+
+Build directory.
+
+=item I<--prefix=DIR>
+
+Change directory to run tests in. Default is 'st'.
+
+=item I<--immediate>
+
+Show errors as soon as they happen rather than at the end of the test run.
+		
+=item I<--target samba4|samba3|win>
+
+Specify test target against which to run. Default is 'samba4'.
+
+=item I<--quick>
+
+Run only a limited number of tests. Intended to run in about 30 seconds on 
+moderately recent systems.
+		
+=item I<--socket-wrapper>
+
+Use socket wrapper library for communication with server. Only works 
+when the server is running locally.
+
+Will prevent TCP and UDP ports being opened on the local host but 
+(transparently) redirects these calls to use unix domain sockets.
+
+=item I<--expected-failures>
+
+Specify a file containing a list of tests that are expected to fail. Failures for 
+these tests will be counted as successes, successes will be counted as failures.
+
+The format for the file is, one entry per line:
+
+TESTSUITE-NAME/TEST-NAME
+
+=item I<--one>
+
+Abort as soon as one test fails.
+
+=back
+
+=head1 ENVIRONMENT
+
+=over 4
+
+=item I<SMBD_VALGRIND>
+
+=item I<TORTURE_MAXTIME>
+
+=item I<VALGRIND>
+
+=item I<TEST_LDAP>
+
+=item I<TLS_ENABLED>
+
+=item I<srcdir>
+
+=back
+
+=head1 LICENSE
+
+selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
+
+=head1 AUTHOR
+
+Jelmer Vernooij
+
+=cut
+
 use strict;
 use warnings;
 
@@ -10,86 +111,163 @@ use File::Spec;
 use Getopt::Long;
 use POSIX;
 use Cwd;
+use lib "$RealBin";
+use Samba4;
+use SocketWrapper;
 
-sub slapd_start($$) {
-	my ($conf, $uri) = @_;
-    my $oldpath = $ENV{PATH};
-    $ENV{PATH} = "/usr/local/sbin:/usr/sbin:/sbin:$ENV{PATH}";
-	# running slapd in the background means it stays in the same process group, so it can be
-	# killed by timelimit
-    system("slapd -d0 -f $conf -h $uri &");
-    $ENV{PATH} = $oldpath;
-    return $? >> 8;
-}
+my $opt_help = 0;
+my $opt_target = "samba4";
+my $opt_quick = 0;
+my $opt_socket_wrapper = 0;
+my $opt_socket_wrapper_pcap = undef;
+my $opt_one = 0;
+my $opt_immediate = 0;
+my $opt_expected_failures = undef;
+my $opt_verbose = 0;
 
-sub smbd_check_or_start($$$$$$) 
+my $srcdir = ".";
+my $builddir = ".";
+my $prefix = "st";
+
+my $suitesfailed = [];
+my $start = time();
+my @expected_failures = ();
+
+my $statistics = {
+	SUITES_FAIL => 0,
+	SUITES_OK => 0,
+
+	TESTS_UNEXPECTED_OK => 0,
+	TESTS_EXPECTED_OK => 0,
+	TESTS_UNEXPECTED_FAIL => 0,
+	TESTS_EXPECTED_FAIL => 0,
+	TESTS_ERROR => 0
+};
+
+sub expecting_failure($)
 {
-	my ($bindir, $test_fifo, $test_log, $socket_wrapper_dir, $max_time, $conffile) = @_;
-	return 0 if ( -p $test_fifo );
+	my $fullname = shift;
 
-	if (defined($socket_wrapper_dir)) {
-		if ( -d $socket_wrapper_dir ) {
-			unlink <$socket_wrapper_dir/*>;
-		} else {
-			mkdir($socket_wrapper_dir);
-		}
+	foreach (@expected_failures) {
+		return 1 if $fullname =~ /^$_$/;
 	}
 
-	unlink($test_fifo);
-	system("mkfifo $test_fifo");
-
-	unlink($test_log);
-	
-	my $valgrind = "";
-	if (defined($ENV{SMBD_VALGRIND})) {
-		$valgrind = $ENV{SMBD_VALGRIND};
-	} 
-
-	print "STARTING SMBD...";
-	my $pid = fork();
-	if ($pid == 0) {
-		my $ret = system("$valgrind $bindir/smbd --maximum-runtime=$max_time -s $conffile -M single -i --leak-report-full < $test_fifo > $test_log");
-		open LOG, ">>$test_log";
-		if ($? == -1) {
-			print LOG "Unable to start smbd: $ret: $!\n";
-			print "Unable to start smbd: $ret: $!\n";
-			exit 1;
-		}
-		unlink($test_fifo);
-		unlink(<$socket_wrapper_dir/*>) if (defined($socket_wrapper_dir) and -d $socket_wrapper_dir);
-		my $exit = $? >> 8;
-		if ( $ret == 0 ) {
-			print "smbd exits with status $exit\n";
-			print LOG "smbd exits with status $exit\n";
-		} elsif ( $ret & 127 ) {
-			print "smbd got signal ".($ret & 127)." and exits with $exit!\n";
-			print LOG "smbd got signal".($ret & 127). " and exits with $exit!\n";
-		} else {
-			$ret = $? >> 8;
-			print "smbd failed with status $exit!\n";
-			print LOG "smbd failed with status $exit!\n";
-		}
-		close(LOG);
-		exit $exit;
-	}
-	print "DONE\n";
-
-	return $pid;
+	return 0;
 }
 
-sub teststatus($$) {
-	my ($name, $failed) = @_;
+sub run_test_buildfarm($$$$)
+{
+	my ($name, $cmd, $i, $suitestotal) = @_;
+	print "--==--==--==--==--==--==--==--==--==--==--\n";
+	print "Running test $name (level 0 stdout)\n";
+	print "--==--==--==--==--==--==--==--==--==--==--\n";
+	system("date");
 
-	print "TEST STATUS: $failed failures\n";
-	if ($failed > 0) {
-print <<EOF	    
-************************
-*** TESTSUITE FAILED ***
-************************
-EOF
-;
+	my $expected_ret = 1;
+	my $open_tests = {};
+	open(RESULT, "$cmd|");
+	while (<RESULT>) { 
+		print;
+		if (/^test: (.+)\n/) {
+			$open_tests->{$1} = 1;
+		} elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
+			my $result = $1;
+			if ($1 eq "success") {
+				delete $open_tests->{$2};
+				if (expecting_failure("$name/$2")) {
+					$statistics->{TESTS_UNEXPECTED_OK}++;
+				} else {
+					$statistics->{TESTS_EXPECTED_OK}++;
+				}
+			} elsif ($1 eq "failure") {
+				delete $open_tests->{$2};
+				if (expecting_failure("$name/$2")) {
+					$statistics->{TESTS_EXPECTED_FAIL}++;
+					$expected_ret = 0;
+				} else {
+					$statistics->{TESTS_UNEXPECTED_FAIL}++;
+				}
+			} elsif ($1 eq "skip") {
+				delete $open_tests->{$2};
+			} elsif ($1 eq "error") {
+				$statistics->{TESTS_ERROR}++;
+				delete $open_tests->{$2};
+			}
+		}
+	}
+	print "COMMAND: $cmd\n";
+	foreach (keys %$open_tests) {
+		print "$_ was started but never finished!\n";		
+		$statistics->{TESTS_ERROR}++;
+	}
+	my $ret = close(RESULT);
+
+	print "==========================================\n";
+	if ($ret == $expected_ret) {
+		print "TEST PASSED: $name\n";
+	} else {
+		print "TEST FAILED: $name (status $ret)\n";
+	}
+	print "==========================================\n";
+}
+
+my $test_output = {};
+sub run_test_plain($$$$)
+{
+	my ($name, $cmd, $i, $totalsuites) = @_;
+	my $err = "";
+	if ($#$suitesfailed+1 > 0) { $err = ", ".($#$suitesfailed+1)." errors"; }
+	printf "[$i/$totalsuites in " . (time() - $start)."s$err] $name\n";
+	open(RESULT, "$cmd 2>&1|");
+	my $expected_ret = 1;
+	my $open_tests = {};
+	$test_output->{$name} = "";
+	while (<RESULT>) { 
+		$test_output->{$name}.=$_;
+		print if ($opt_verbose);
+		if (/^test: (.+)\n/) {
+			$open_tests->{$1} = 1;
+		} elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
+			my $result = $1;
+			if ($1 eq "success") {
+				delete $open_tests->{$2};
+				if (expecting_failure("$name/$2")) {
+					$statistics->{TESTS_UNEXPECTED_OK}++;
+				} else {
+					$statistics->{TESTS_EXPECTED_OK}++;
+				}
+			} elsif ($1 eq "failure") {
+				delete $open_tests->{$2};
+				if (expecting_failure("$name/$2")) {
+					$statistics->{TESTS_EXPECTED_FAIL}++;
+					$expected_ret = 0;
+				} else {
+					$statistics->{TESTS_UNEXPECTED_FAIL}++;
+				}
+			} elsif ($1 eq "skip") {
+				delete $open_tests->{$2};
+			} elsif ($1 eq "error") {
+				$statistics->{TESTS_ERROR}++;
+				delete $open_tests->{$2};
+			}
+		}
+	}
+	$test_output->{$name}.="COMMAND: $cmd\n";
+	foreach (keys %$open_tests) {
+		$test_output->{$name}.="$_ was started but never finished!\n";		
+		$statistics->{TESTS_ERROR}++;
+	}
+	my $ret = close(RESULT);
+	if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
+		print "$test_output->{$name}\n";
+	}
+	if ($ret != $expected_ret) {
+		push(@$suitesfailed, $name);
+		$statistics->{SUITES_FAIL}++;
+		exit(1) if ($opt_one);
+	} else {
+		$statistics->{SUITES_OK}++;
 	}
-	exit $failed;
 }
 
 sub ShowHelp()
@@ -97,51 +275,60 @@ sub ShowHelp()
 	print "Samba test runner
 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 
-Usage: $Script PREFIX
+Usage: $Script [OPTIONS] PREFIX
 
 Generic options:
  --help                     this help page
+
+Paths:
+ --prefix=DIR               prefix to run tests in [st]
+ --srcdir=DIR               source directory [.]
+ --builddir=DIR             output directory [.]
+
+Target Specific:
  --target=samba4|samba3|win Samba version to target
+ --socket-wrapper-pcap=FILE save traffic to pcap file
  --socket-wrapper           enable socket wrapper
+ --expected-failures=FILE   specify list of tests that is guaranteed to fail
+
+Behaviour:
  --quick                    run quick overall test
  --one                      abort when the first test fails
+ --immediate                print test output for failed tests during run
+ --verbose                  be verbose
 ";
 	exit(0);
 }
 
-my $opt_help = 0;
-my $opt_target = "samba4";
-my $opt_quick = 0;
-my $opt_socket_wrapper = 0;
-my $opt_one = 0;
-
 my $result = GetOptions (
 	    'help|h|?' => \$opt_help,
-		'target' => \$opt_target,
+		'target=s' => \$opt_target,
+		'prefix=s' => \$prefix,
 		'socket-wrapper' => \$opt_socket_wrapper,
+		'socket-wrapper-pcap=s' => \$opt_socket_wrapper_pcap,
 		'quick' => \$opt_quick,
-		'one' => \$opt_one
+		'one' => \$opt_one,
+		'immediate' => \$opt_immediate,
+		'expected-failures=s' => \$opt_expected_failures,
+		'srcdir=s' => \$srcdir,
+		'builddir=s' => \$builddir,
+		'verbose' => \$opt_verbose
 	    );
 
-if (not $result) {
-	exit(1);
-}
+exit(1) if (not $result);
 
 ShowHelp() if ($opt_help);
-ShowHelp() if ($#ARGV < 0);
 
-my $prefix = shift;
+my $tests = shift;
 
 my $torture_maxtime = $ENV{TORTURE_MAXTIME};
 unless (defined($torture_maxtime)) {
 	$torture_maxtime = 1200;
 }
 
-# disable rpc validation when using valgrind - its way too slow
-my $valgrind = $ENV{VALGRIND};
-my $validate = undef;
-unless (defined($valgrind)) {
-	$validate = "validate";
+# quick hack to disable rpc validation when using valgrind - its way too slow
+unless (defined($ENV{VALGRIND})) {
+	$ENV{VALIDATE} = "validate";
 }
 
 my $old_pwd = "$RealBin/../..";
@@ -150,10 +337,6 @@ my $ldap = (defined($ENV{TEST_LDAP}) and ($ENV{TEST_LDAP} eq "yes"))?1:0;
 $prefix =~ s+//+/+;
 $ENV{PREFIX} = $prefix;
 
-my $srcdir = "$RealBin/../..";
-if (defined($ENV{srcdir})) {
-	$srcdir = $ENV{srcdir};
-}
 $ENV{SRCDIR} = $srcdir;
 
 my $bindir = "$srcdir/bin";
@@ -161,6 +344,8 @@ my $setupdir = "$srcdir/setup";
 my $testsdir = "$srcdir/script/tests";
 
 my $tls_enabled = not $opt_quick;
+my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and 
+                      ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 
 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
@@ -173,14 +358,12 @@ if (defined($ENV{LD_LIBRARY_PATH})) {
 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 
+my @torture_options = ();
+
+my $testenv_vars = {};
+
 if ($opt_target eq "samba4") {
-	print "PROVISIONING...";
-	open(IN, "$RealBin/mktestsetup.sh $prefix|") or die("Unable to setup");
-	while (<IN>) {
-		next unless (/^([A-Z_]+)=(.*)$/);
-		$ENV{$1} = $2;
-	}
-	close(IN);
+	$testenv_vars = Samba4::provision($prefix);
 } elsif ($opt_target eq "win") {
 	die ("Windows tests will not run without root privileges.") 
 		if (`whoami` ne "root");
@@ -196,77 +379,82 @@ if ($opt_target eq "samba4") {
 	die ("$ENV{WINTESTCONF} could not be read.") if (! -r $ENV{WINTESTCONF});
 
 	$ENV{WINTEST_DIR}="$ENV{SRCDIR}/script/tests/win";
+} elsif ($opt_target eq "none") {
 } else {
 	die("unknown target `$opt_target'");
 }
 
-my $socket_wrapper_dir = undef;
+foreach (keys %$testenv_vars) { $ENV{$_} = $testenv_vars->{$_}; }
 
-if ( $opt_socket_wrapper) 
+if ($opt_socket_wrapper_pcap) {
+	$ENV{SOCKET_WRAPPER_PCAP_FILE} = $opt_socket_wrapper_pcap;
+	# Socket wrapper pcap implies socket wrapper
+	$opt_socket_wrapper = 1;
+}
+
+my $socket_wrapper_dir;
+if ($opt_socket_wrapper) 
 {
-	$socket_wrapper_dir = "$prefix/w";
-	$ENV{SOCKET_WRAPPER_DIR} = $socket_wrapper_dir;
-	print "SOCKET_WRAPPER_DIR=$ENV{SOCKET_WRAPPER_DIR}\n";
-} else {
-	print "NOT USING SOCKET_WRAPPER\n";
+	$socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w");
+	print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 }
 
 # Start slapd before smbd
 if ($ldap) {
-    slapd_start($ENV{SLAPD_CONF}, $ENV{LDAPI_ESCAPE}) or die("couldn't start slapd");
+	Samba4::slapd_start($ENV{SLAPD_CONF}, $ENV{LDAPI_ESCAPE}) or die("couldn't start slapd");
+
     print "LDAP PROVISIONING...";
-    system("$bindir/smbscript $setupdir/provision $ENV{PROVISION_OPTIONS} --ldap-backend=$ENV{LDAPI}") or
-		die("LDAP PROVISIONING failed: $bindir/smbscript $setupdir/provision $ENV{PROVISION_OPTIONS} --ldap-backend=$ENV{LDAPI}");
+	Samba4::provision_ldap($bindir, $setupdir);
 
     # LDAP is slow
 	$torture_maxtime *= 2;
 }
 
+if (defined($opt_expected_failures)) {
+	open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
+	while (<KNOWN>) { 
+		chomp; 
+		s/([ \t]+)\#(.*)$//;
+		push (@expected_failures, $_); }
+	close(KNOWN);
+}
+
 my $test_fifo = "$prefix/smbd_test.fifo";
 
 $ENV{SMBD_TEST_FIFO} = $test_fifo;
 $ENV{SMBD_TEST_LOG} = "$prefix/smbd_test.log";
 
-$ENV{SOCKET_WRAPPER_DEFAULT_IFACE} = 1;
+SocketWrapper::set_default_iface(1);
 my $max_time = 5400;
 if (defined($ENV{SMBD_MAX_TIME})) {
 	$max_time = $ENV{SMBD_MAX_TIME};
 }
-smbd_check_or_start($bindir, $test_fifo, $ENV{SMBD_TEST_LOG}, $socket_wrapper_dir, $max_time, $ENV{CONFFILE});
+Samba4::smbd_check_or_start($bindir, $test_fifo, $ENV{SMBD_TEST_LOG}, 
+	                $socket_wrapper_dir, $max_time, $ENV{CONFFILE});
 
-$ENV{SOCKET_WRAPPER_DEFAULT_IFACE} = 6;
-$ENV{TORTURE_INTERFACES} = '127.0.0.6/8,127.0.0.7/8,127.0.0.8/8,127.0.0.9/8,127.0.0.10/8,127.0.0.11/8';
+SocketWrapper::set_default_iface(6);
 
-my @torture_options = ("--option=interfaces=$ENV{TORTURE_INTERFACES} $ENV{CONFIGURATION}");
+my $interfaces = join(',', ("127.0.0.6/8", 
+		                 "127.0.0.7/8",
+						 "127.0.0.8/8",
+						 "127.0.0.9/8",
+						 "127.0.0.10/8",
+						 "127.0.0.11/8"));
+
+push (@torture_options, "--option=interfaces=$interfaces");
+push (@torture_options, $ENV{CONFIGURATION});
 # ensure any one smbtorture call doesn't run too long
 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 push (@torture_options, "--target=$opt_target");
-push (@torture_options, "--option=torture:progress=no") 
-	if (defined($ENV{RUN_FROM_BUILD_FARM}) and $ENV{RUN_FROM_BUILD_FARM} eq "yes");
+push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
+push (@torture_options, "--format=subunit");
+push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 
 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 
-my $start = time();
-
 open(DATA, ">$test_fifo");
 
-# give time for nbt server to register its names
-print "delaying for nbt name registration\n";
-sleep(4);
-
-# This will return quickly when things are up, but be slow if we need to wait for (eg) SSL init 
-system("bin/nmblookup $ENV{CONFIGURATION} $ENV{SERVER}");
-system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{SERVER}");
-system("bin/nmblookup $ENV{CONFIGURATION} $ENV{SERVER}");
-system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{NETBIOSNAME}");
-system("bin/nmblookup $ENV{CONFIGURATION} $ENV{NETBIOSNAME}");
-system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{NETBIOSNAME}");
-
-# start off with 0 failures
-$ENV{failed} = 0;
-my $totalfailed = 0;
-
 my @todo = ();
 
 if ($opt_target eq "win") {
@@ -283,27 +471,35 @@ if ($opt_target eq "win") {
 			$name =~ s/\n//g;
 			my $cmdline = <IN>;
 			$cmdline =~ s/\n//g;
-			push (@todo, [$name, $cmdline]);
+			push (@todo, [$name, $cmdline]) 
+				if (not defined($tests) or $name =~ /$tests/);
 		} else {
 			print;
 		}
 	}
-	close(IN);
+	close(IN) or die("Error creating recipe");
 }
 
-my $total = $#todo + 1;
+Samba4::wait_for_start();
+
+# start off with 0 failures
+$ENV{failed} = 0;
+
+my $suitestotal = $#todo + 1;
 my $i = 0;
 $| = 1;
 
+delete $ENV{DOMAIN};
+
 foreach (@todo) {
-	$i = $i + 1;
-	my $err = "";
-	if ($totalfailed > 0) { $err = ", $totalfailed errors"; }
-	printf "[$i/$total in " . (time() - $start)."s$err] $$_[0]\n";
-	my $ret = system("$$_[1] >/dev/null 2>/dev/null");
-	if ($ret != 0) {
-		$totalfailed++;
-		exit(1) if ($opt_one);
+	$i++;
+	my $cmd = $$_[1];
+	$cmd =~ s/([\(\)])/\\$1/g;
+	my $name = $$_[0];
+	if ($from_build_farm) {
+		run_test_buildfarm($name, $cmd, $i, $suitestotal);
+	} else {
+		run_test_plain($name, $cmd, $i, $suitestotal);
 	}
 }
 
@@ -311,6 +507,8 @@ print "\n";
 
 close(DATA);
 
+sleep(2);
+
 my $failed = $? >> 8;
 
 if (-f "$ENV{PIDDIR}/smbd.pid" ) {
@@ -319,15 +517,37 @@ if (-f "$ENV{PIDDIR}/smbd.pid" ) {
 	close(IN);
 }
 
-if ($ldap) {
-    open(IN, "<$ENV{PIDDIR}/slapd.pid") or die("unable to open slapd pid file");
-	kill 9, <IN>;
-	close(IN);
-}
+Samba4::slapd_stop() if ($ldap);
 
-my $end=time();
-print "DURATION: " . ($end-$start). " seconds\n";
-print "$totalfailed failures\n";
+my $end = time();
+my $duration = ($end-$start);
+my $numfailed = $#$suitesfailed+1;
+if ($numfailed == 0) {
+	my $ok = $statistics->{TESTS_EXPECTED_OK} + $statistics->{TESTS_EXPECTED_FAIL};
+	print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
+} else {
+
+	unless ($from_build_farm) {
+		if (not $opt_immediate and not $opt_verbose) {
+			foreach (@$suitesfailed) {
+				print "===============================================================================\n";
+				print "FAIL: $_\n";
+				print $test_output->{$_};
+				print "\n";
+			}
+		}
+
+		print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
+	} else {
+		print <<EOF	    
+************************
+*** TESTSUITE FAILED ***
+************************
+EOF
+;
+	}
+}
+print "DURATION: $duration seconds\n";
 
 # if there were any valgrind failures, show them
 foreach (<$prefix/valgrind.log*>) {
@@ -340,6 +560,4 @@ foreach (<$prefix/valgrind.log*>) {
 	}
 }
 
-teststatus($Script, $failed);
-
-exit $failed;
+exit $numfailed;

source/script/tests/selftest.sh

@@ -1,290 +0,0 @@
-#!/bin/sh
-# Bootstrap Samba and run a number of tests against it.
-
-if [ $# -lt 1 ]
-then
-	echo "$0 PREFIX TESTS"
-	exit
-fi
-
-ARG0=$0
-ARG1=$1
-ARG2=$2
-ARG3=$3
-
-if [ -z "$TORTURE_MAXTIME" ]; then
-    TORTURE_MAXTIME=1200
-fi
-
-# disable rpc validation when using valgrind - its way too slow
-if [ -z "$VALGRIND" ]; then
-    VALIDATE="validate";
-else
-    VALIDATE="";
-fi
-
-OLD_PWD=`pwd`
-PREFIX=$ARG1
-PREFIX=`echo $PREFIX | sed s+//+/+`
-export PREFIX
-
-# allow selection of the test lists
-TESTS=$ARG2
-
-if [ $TESTS = "all" ]; then
-    TLS_ENABLED="yes"
-else
-    TLS_ENABLED="no"
-fi
-export TLS_ENABLED
-
-LD_LDB_MODULE_PATH=$OLD_PWD/bin/modules/ldb
-export LD_LDB_MODULE_PATH
-
-LD_SAMBA_MODULE_PATH=$OLD_PWD/bin/modules
-export LD_SAMBA_MODULE_PATH
-
-LD_LIBRARY_PATH=$OLD_PWD/bin/shared:$LD_LIBRARY_PATH
-export LD_LIBRARY_PATH
-
-PKG_CONFIG_PATH=$OLD_PWD/bin/pkgconfig:$PKG_CONFIG_PATH
-export PKG_CONFIG_PATH
-
-incdir=`dirname $ARG0`
-echo -n "PROVISIONING..."
-eval `$incdir/mktestsetup.sh $PREFIX || exit 1`
-echo "DONE"
-
-export KRB5_CONFIG
-export PREFIX_ABS
-export TEST_DATA_PREFIX
-export CONFIGURATION
-export CONFFILE
-export PIDDIR
-export AUTH
-export SERVER
-export NETBIOSNAME
-
-PATH=bin:$PATH
-export PATH
-
-DO_SOCKET_WRAPPER=$ARG3
-if [ x"$DO_SOCKET_WRAPPER" = x"SOCKET_WRAPPER" ];then
-	SOCKET_WRAPPER_DIR="$PREFIX/w"
-	export SOCKET_WRAPPER_DIR
-	echo "SOCKET_WRAPPER_DIR=$SOCKET_WRAPPER_DIR"
-else
-	echo "NOT USING SOCKET_WRAPPER"
-fi
-
-incdir=`dirname $ARG0`
-. $incdir/test_functions.sh
-
-#Start slapd before smbd
-if [ x"$TEST_LDAP" = x"yes" ]; then
-    if test -z "$FEDORA_DS_PREFIX"; then
-	slapd_start || exit 1;
-    else
-	fedora_ds_start || exit 1;
-    fi
-    echo -n "LDAP PROVISIONING..."
-    $srcdir/bin/smbscript $srcdir/setup/provision $PROVISION_OPTIONS "$PROVISION_ACI" --ldap-backend=$LDAP_URI || {
-	echo "LDAP PROVISIONING failed: $srcdir/bin/smbscript $srcdir/setup/provision $PROVISION_OPTIONS $PROVISION_ACI --ldap-backend=$LDAP_URI"
-	exit 1;
-    }
-    #LDAP is slow
-    TORTURE_MAXTIME=`expr $TORTURE_MAXTIME '*' 2`
-fi
-
-SMBD_TEST_FIFO="$PREFIX/smbd_test.fifo"
-export SMBD_TEST_FIFO
-SMBD_TEST_LOG="$PREFIX/smbd_test.log"
-export SMBD_TEST_LOG
-
-SOCKET_WRAPPER_DEFAULT_IFACE=1
-export SOCKET_WRAPPER_DEFAULT_IFACE
-smbd_check_or_start
-
-SOCKET_WRAPPER_DEFAULT_IFACE=6
-export SOCKET_WRAPPER_DEFAULT_IFACE
-TORTURE_INTERFACES='127.0.0.6/8,127.0.0.7/8,127.0.0.8/8,127.0.0.9/8,127.0.0.10/8,127.0.0.11/8'
-TORTURE_OPTIONS="--option=interfaces=$TORTURE_INTERFACES $CONFIGURATION"
-# ensure any one smbtorture call doesn't run too long
-TORTURE_OPTIONS="$TORTURE_OPTIONS --maximum-runtime=$TORTURE_MAXTIME"
-TORTURE_OPTIONS="$TORTURE_OPTIONS --target=samba4"
-export TORTURE_OPTIONS
-
-if [ x"$RUN_FROM_BUILD_FARM" = x"yes" ];then
-	TORTURE_OPTIONS="$TORTURE_OPTIONS --option=torture:progress=no"
-fi
-
- runtest() {
-	if [ -z "$PREFIX" ]; then
-	    PREFIX=test_prefix
-	    mkdir -p $PREFIX
-	fi
-	name=$1
-	shift 1
-	cmdline="$*"
-
-	SMBD_IS_UP="no"
-
-	shname=`echo $name | \
-	sed -e 's%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g'`
-
-	UNIQUE_PID=`/bin/sh -c 'echo $$'`
-	TEST_LOG="$PREFIX/test_log.${UNIQUE_PID}"
-	TEST_PCAP="$PREFIX/test_${shname}_${UNIQUE_PID}.pcap"
-	trap "rm -f $TEST_LOG $TEST_PCAP" EXIT
-
-        if [ -n "$SMBD_TEST_LOG" -a -z "$smbd_log_size" ]; then
-	    smbd_log_size=`wc -l < $SMBD_TEST_LOG`;
-	fi
-
-	if [ x"$RUN_FROM_BUILD_FARM" = x"yes" ];then
-		echo "--==--==--==--==--==--==--==--==--==--==--"
-		echo "Running test $name (level 0 stdout)"
-		echo "--==--==--==--==--==--==--==--==--==--==--"
-		date
-		echo "Testing $name"
-	else
-	        nf="`expr $failed + $totalfailed`";
-		if [ "$nf" = "0" ]; then 
-		    echo "[$current/$total] Testing $name"
-		else 
-		    echo "[$current/$total, $nf failures] Testing $name"
-		fi
-	fi
-
-	smbd_check_only && SMBD_IS_UP="yes"
-	if [ x"$SMBD_IS_UP" != x"yes" ];then
-		if [ x"$RUN_FROM_BUILD_FARM" = x"yes" ];then
-			echo "SMBD is down! Skipping: $cmdline"
-			echo "=========================================="
-			echo "TEST SKIPPED: $name (reason SMBD is down)"
-			echo "=========================================="
-   		else
-			echo "TEST SKIPPED: $name (reason SMBD is down)"
-		fi
-		return 1
-	fi
-
-	if [ x"$MAKE_TEST_ENABLE_PCAP" = x"yes" ];then
-		SOCKET_WRAPPER_PCAP_FILE=$TEST_PCAP
-		export SOCKET_WRAPPER_PCAP_FILE
-	fi
-
-	( $cmdline > $TEST_LOG 2>&1 )
-	status=$?
-	# show any additional output from smbd that has happened in this test
-	smbd_have_test_log && {		    
-	    new_log_size=`wc -l < $SMBD_TEST_LOG`;
-	    test "$new_log_size" = "$smbd_log_size" || {
-		echo "SMBD OUTPUT:";
-		incr_log_size=`expr $new_log_size - $smbd_log_size`;
-		tail -$incr_log_size $SMBD_TEST_LOG;
-		smbd_log_size=$new_log_size;
-	    }
-	}
-	if [ x"$status" != x"0" ]; then
-		echo "TEST OUTPUT:"
-		cat $TEST_LOG;
-		rm -f $TEST_LOG;
-		if [ x"$MAKE_TEST_ENABLE_PCAP" = x"yes" ];then
-			echo "TEST PCAP: $TEST_PCAP"
-		fi
-		if [ x"$RUN_FROM_BUILD_FARM" = x"yes" ];then
-			echo "=========================================="
-			echo "TEST FAILED: $name (status $status)"
-			echo "=========================================="
-   		else
-			echo "TEST FAILED: $cmdline (status $status)"
-		fi
-		trap "" EXIT
-		return 1;
-	fi
-	rm -f $TEST_LOG;
-	if [ x"$MAKE_TEST_KEEP_PCAP" = x"yes" ];then
-		echo "TEST PCAP: $TEST_PCAP"
-	else
-		rm -f $TEST_PCAP;
-	fi
-	if [ x"$RUN_FROM_BUILD_FARM" = x"yes" ];then
-		echo "ALL OK: $cmdline"
-		echo "=========================================="
-		echo "TEST PASSED: $name"
-		echo "=========================================="
-	fi
-	trap "" EXIT
-	return 0;
-}
-
-START=`date`
-(
- # give time for nbt server to register its names
- echo delaying for nbt name registration
- sleep 4
- # This will return quickly when things are up, but be slow if we need to wait for (eg) SSL init 
- bin/nmblookup $CONFIGURATION $SERVER
- bin/nmblookup $CONFIGURATION -U $SERVER $SERVER
- bin/nmblookup $CONFIGURATION $SERVER
- bin/nmblookup $CONFIGURATION -U $SERVER $NETBIOSNAME
- bin/nmblookup $CONFIGURATION $NETBIOSNAME
- bin/nmblookup $CONFIGURATION -U $SERVER $NETBIOSNAME
-
-# start off with 0 failures
-totalfailed=0
-export totalfailed
-
-. script/tests/tests_$TESTS.sh > $PREFIX/recipe
-total=`grep "TEST --" $PREFIX/recipe | wc -l`
-current=0
-cat $PREFIX/recipe | (
- 	while read LINE
-	do
-		if [ "$LINE" = "-- TEST --" ]; then
-			read NAME
-			read CMDLINE
-			current=`expr $current + 1`
-			runtest "$NAME" "$CMDLINE" || totalfailed=`expr $totalfailed + $?`
-		else
-			echo "$LINE"
-		fi
- 	done
-	exit $totalfailed
-)
-exit $?
-) 9>$SMBD_TEST_FIFO
-totalfailed=$?
-
-smbd_PID=`cat $PIDDIR/smbd.pid`
-waitforpid $smbd_PID 20 || {
-    echo "smbd process $1 took more than 20 seconds to exit, killing"
-    kill -9 $smbd_PID
-}
-
-if [ "$TEST_LDAP"x = "yesx" ]; then
-    if test -z "$FEDORA_DS_PREFIX"; then
-	kill `cat $PIDDIR/slapd.pid`
-    else
-	$LDAPDIR/slapd-samba4/stop-slapd
-    fi
-fi
-
-END=`date`
-echo "START: $START ($ARG0)";
-echo "END:   $END ($ARG0)";
-
-# if there were any valgrind failures, show them
-count=`find $PREFIX -name 'valgrind.log*' | wc -l`
-if [ "$count" != 0 ]; then
-    for f in $PREFIX/valgrind.log*; do
-	if [ -s $f ] && grep -v DWARF2.CFI.reader $f > /dev/null; then
-	    echo "VALGRIND FAILURE";
-	    totalfailed=`expr $totalfailed + 1`
-	    cat $f
-	fi
-    done
-fi
-
-teststatus $ARG0 $totalfailed

source/script/tests/test_ejs.sh

@@ -24,16 +24,11 @@ PATH=bin:$PATH
 export PATH
 
 testit "base.js" $SCRIPTDIR/base.js $CONFIGURATION
-
 testit "samr.js" "$SCRIPTDIR/samr.js" $CONFIGURATION ncalrpc: -U$USERNAME%$PASSWORD
 testit "echo.js" "$SCRIPTDIR/echo.js" $CONFIGURATION ncalrpc: -U$USERNAME%$PASSWORD
-
 testit "ejsnet.js" $SCRIPTDIR/ejsnet.js $CONFIGURATION -U$USERNAME%$PASSWORD $DOMAIN ejstestuser
-
 testit "ldb.js" $SCRIPTDIR/ldb.js `pwd` $CONFIGURATION
-
 testit "samba3sam.js" $SCRIPTDIR/samba3sam.js $CONFIGURATION `pwd` $DATADIR/samba3/
-
 testit "winreg" scripting/bin/winreg $CONFIGURATION ncalrpc: 'HKLM' -U$USERNAME%$PASSWORD
 
 testok $0 $failed

source/script/tests/test_ldap.sh

@@ -15,7 +15,6 @@ PASSWORD="$3"
 incdir=`dirname $0`
 . $incdir/test_functions.sh
 
-
 p=ldap
 for options in "" "--option=socket:testnonblock=true" "-U$USERNAME%$PASSWORD --option=socket:testnonblock=true" "-U$USERNAME%$PASSWORD"; do
     testit "TESTING PROTOCOL $p with options $options" ../testprogs/blackbox/test_ldb.sh $p $SERVER $options
@@ -37,11 +36,11 @@ done
 test "$TORTURE_QUICK" = "yes" || {
    LDBDIR=lib/ldb
    export LDBDIR
-   testit "ldb tests" $LDBDIR/tests/test-tdb.sh
+   testit "ldb" $LDBDIR/tests/test-tdb.sh
 }
 
 SCRIPTDIR=../testprogs/ejs
 
-testit "ejs ldap test" $SCRIPTDIR/ldap.js $CONFIGURATION $SERVER -U$USERNAME%$PASSWORD 
+testit "ejs ldap" $SCRIPTDIR/ldap.js $CONFIGURATION $SERVER -U$USERNAME%$PASSWORD
 
 testok $0 $failed

source/script/tests/test_nbt.sh

@@ -23,9 +23,7 @@ testit "nmblookup $SERVER" bin/nmblookup $TORTURE_OPTIONS $SERVER
 
 NBT_TESTS="NBT-REGISTER NBT-WINS"
 NBT_TESTS="$NBT_TESTS NBT-WINSREPLICATION"
-# if [ "$TORTURE_QUICK"x != "yes"x ]; then
-# 	NBT_TESTS="$NBT_TESTS NBT-WINSREPLICATION-OWNED"
-# fi
+# NBT_TESTS="$NBT_TESTS NBT-WINSREPLICATION-OWNED"
 NBT_TESTS="$NBT_TESTS NET-API-LOOKUP NET-API-LOOKUPHOST NET-API-LOOKUPPDC"
 
 for f in $NBT_TESTS; do

source/script/tests/testenv.pl

@@ -0,0 +1,32 @@
+#!/usr/bin/perl
+
+use FindBin qw($RealBin);
+use lib "$RealBin";
+
+use Samba4;
+use SocketWrapper;
+
+my $vars = Samba4::provision("st");
+foreach (keys %$vars) { $ENV{$_} = $vars->{$_}; }
+SocketWrapper::set_default_iface(1);
+my $test_fifo = "st/smb_test.fifo";
+my $socket_wrapper_dir = SocketWrapper::setup_dir("$vars->{PREFIX_ABS}/w");
+Samba4::smbd_check_or_start("bin", $test_fifo, $ENV{SMBD_TEST_LOG}, $socket_wrapper_dir, undef, $ENV{CONFFILE});
+SocketWrapper::set_default_iface(6);
+my $interfaces = join(',', ("127.0.0.6/8", 
+		                 "127.0.0.7/8",
+						 "127.0.0.8/8",
+						 "127.0.0.9/8",
+						 "127.0.0.10/8",
+						 "127.0.0.11/8"));
+
+push (@torture_options, "--option=interfaces=$interfaces", 
+	                    $ENV{CONFIGURATION}, 
+						"--target=samba4");
+
+$ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
+
+open(DATA, ">$test_fifo");
+Samba4::wait_for_start();
+system("xterm");
+close(DATA);

source/script/tests/tests_net.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
- $SRCDIR/script/tests/test_net.sh $SERVER $USERNAME $PASSWORD $DOMAIN

source/script/tests/tests_quick.sh

@@ -1,6 +1,4 @@
 #!/bin/sh
-TORTURE_OPTIONS="$TORTURE_OPTIONS --option=torture:quick=yes"
-export TORTURE_OPTIONS
 TORTURE_QUICK="yes"
 export TORTURE_QUICK