1
0
mirror of https://github.com/samba-team/samba.git synced 2024-12-23 17:34:34 +03:00

r19343: Add support for external scripts/binaries that write results using the

'subunit' protocol. This allows us to easily plug EJS scripts or binaries that
can't depend on -ltorture into smbtorture. The protocol is very simple:

- write "comments" to stderr

Example output on stdout:
test: foo
success: foo
test: bar
success: bar
test: blah
failure: blah [
dummy.c:30: Expression 1 != 2 failed!
]
test: blie
skip: blie [
Iconv support not built in
]

I've already converted the talloc testsuite.
(This used to be commit e1742c14a2)
This commit is contained in:
Jelmer Vernooij 2006-10-16 20:05:19 +00:00 committed by Gerald (Jerry) Carter
parent 3478bf1c23
commit 52e3f69a36
15 changed files with 589 additions and 222 deletions

View File

@ -22,6 +22,7 @@ sub new($$$)
$self->{manpages} = [];
$self->{sbin_progs} = [];
$self->{bin_progs} = [];
$self->{torture_progs} = [];
$self->{static_libs} = [];
$self->{shared_libs} = [];
$self->{installable_shared_libs} = [];
@ -79,6 +80,7 @@ BASEDIR = $self->{config}->{prefix}
BINDIR = $self->{config}->{bindir}
SBINDIR = $self->{config}->{sbindir}
LIBDIR = $self->{config}->{libdir}
TORTUREDIR = $self->{config}->{libdir}/torture
MODULESDIR = $self->{config}->{modulesdir}
INCLUDEDIR = $self->{config}->{includedir}
CONFIGDIR = $self->{config}->{sysconfdir}
@ -364,11 +366,18 @@ sub Binary($$)
my ($self,$ctx) = @_;
my $installdir;
my $localdir;
if (defined($ctx->{INSTALLDIR}) && $ctx->{INSTALLDIR} eq "TORTUREDIR") {
$localdir = "bin/torture";
} else {
$localdir = "bin";
}
if ($self->{duplicate_build}) {
$installdir = "bin/install";
} else {
$installdir = "bin";
$installdir = $localdir;
}
push(@{$self->{all_objs}}, "\$($ctx->{TYPE}_$ctx->{NAME}_FULL_OBJ_LIST)");
@ -378,9 +387,12 @@ sub Binary($$)
push (@{$self->{sbin_progs}}, "$installdir/$ctx->{BINARY}");
} elsif ($ctx->{INSTALLDIR} eq "BINDIR") {
push (@{$self->{bin_progs}}, "$installdir/$ctx->{BINARY}");
} elsif ($ctx->{INSTALLDIR} eq "TORTUREDIR") {
push (@{$self->{torture_progs}}, "$installdir/$ctx->{BINARY}");
}
push (@{$self->{binaries}}, "bin/$ctx->{BINARY}");
push (@{$self->{binaries}}, "$localdir/$ctx->{BINARY}");
$self->_prepare_list($ctx, "OBJ_LIST");
$self->_prepare_list($ctx, "FULL_OBJ_LIST");
@ -390,7 +402,7 @@ sub Binary($$)
if ($self->{duplicate_build}) {
$self->output(<< "__EOD__"
#
bin/$ctx->{BINARY}: \$($ctx->{TYPE}_$ctx->{NAME}_DEPEND_LIST) \$($ctx->{TYPE}_$ctx->{NAME}_FULL_OBJ_LIST)
$localdir/$ctx->{BINARY}: \$($ctx->{TYPE}_$ctx->{NAME}_DEPEND_LIST) \$($ctx->{TYPE}_$ctx->{NAME}_FULL_OBJ_LIST)
\@echo Linking \$\@
\@\$(LD) \$(LDFLAGS) -o \$\@ \$(LOCAL_LINK_FLAGS) \$(INSTALL_LINK_FLAGS) \\
\$\($ctx->{TYPE}_$ctx->{NAME}_LINK_FLAGS)
@ -511,6 +523,7 @@ sub write($$)
$self->output("MANPAGES = ".array2oneperline($self->{manpages})."\n");
$self->output("BIN_PROGS = " . array2oneperline($self->{bin_progs}) . "\n");
$self->output("SBIN_PROGS = " . array2oneperline($self->{sbin_progs}) . "\n");
$self->output("TORTURE_PROGS = " . array2oneperline($self->{torture_progs}) . "\n");
$self->output("BINARIES = " . array2oneperline($self->{binaries}) . "\n");
$self->output("STATIC_LIBS = " . array2oneperline($self->{static_libs}) . "\n");
$self->output("SHARED_LIBS = " . array2oneperline($self->{shared_libs}) . "\n");

View File

@ -88,3 +88,6 @@ _PUBLIC_ const char *dyn_JSDIR = JSDIR;
/** Where to find the winbindd socket */
_PUBLIC_ const char *dyn_WINBINDD_SOCKET_DIR = WINBINDD_SOCKET_DIR;
/** Directory with subunit torture tests */
_PUBLIC_ const char *dyn_TORTUREDIR = TORTUREDIR;

View File

@ -40,3 +40,4 @@ extern const char *dyn_SWATDIR;
extern const char *dyn_JSDIR;
extern const char *dyn_SETUPDIR;
extern const char *dyn_WINBINDD_SOCKET_DIR;
extern const char *dyn_TORTUREDIR;

View File

@ -10,6 +10,7 @@ PATH_FLAGS = -DCONFIGFILE=\"$(CONFIGFILE)\" \
-DCONFIGDIR=\"$(CONFIGDIR)\" -DNCALRPCDIR=\"$(NCALRPCDIR)\" \
-DSWATDIR=\"$(SWATDIR)\" -DPRIVATE_DIR=\"$(PRIVATEDIR)\" \
-DMODULESDIR=\"$(MODULESDIR)\" -DJSDIR=\"$(JSDIR)\" \
-DTORTUREDIR=\"$(TORTUREDIR)\" \
-DSETUPDIR=\"$(SETUPDIR)\" -DWINBINDD_SOCKET_DIR=\"$(WINBINDD_SOCKET_DIR)\"
dynconfig.o: dynconfig.c Makefile

View File

@ -12,3 +12,7 @@ DESCRIPTION = A hierarchical pool based memory system with destructors
# End LIBRARY LIBTALLOC
################################################
[BINARY::LOCAL-TALLOC]
OBJ_FILES = testsuite.o
PRIVATE_DEPENDENCIES = LIBTALLOC
INSTALLDIR = TORTUREDIR

File diff suppressed because it is too large Load Diff

View File

@ -61,6 +61,7 @@ showlayout:
@echo ' jsdir: $(JSDIR)'
@echo ' swatdir: $(SWATDIR)'
@echo ' mandir: $(MANDIR)'
@echo ' torturedir: $(TORTUREDIR)'
@echo ' datadir: $(DATADIR)'
@echo ' winbindd_socket_dir: $(WINBINDD_SOCKET_DIR)'
@ -105,6 +106,7 @@ installdirs:
$(DESTDIR)$(BASEDIR) \
$(DESTDIR)$(BINDIR) \
$(DESTDIR)$(SBINDIR) \
$(DESTDIR)$(TORTUREDIR) \
$(DESTDIR)$(LIBDIR) \
$(DESTDIR)$(MODULESDIR) \
$(DESTDIR)$(MANDIR) \
@ -134,6 +136,13 @@ installbin: $(SBIN_PROGS) $(BIN_PROGS) installdirs
$(DESTDIR)$(LIBDIR) \
$(DESTDIR)$(VARDIR) \
$(BIN_PROGS)
@$(SHELL) $(srcdir)/script/installbin.sh \
$(INSTALLPERMS) \
$(DESTDIR)$(BASEDIR) \
$(DESTDIR)$(TORTUREDIR) \
$(DESTDIR)$(LIBDIR) \
$(DESTDIR)$(VARDIR) \
$(TORTURE_PROGS)
installlib: $(INSTALLABLE_SHARED_LIBS) $(STATIC_LIBS) installdirs
@$(SHELL) $(srcdir)/script/installlib.sh $(DESTDIR)$(LIBDIR) "$(SHLIBEXT)" $(INSTALLABLE_SHARED_LIBS)
@ -166,6 +175,7 @@ uninstallmisc:
uninstallbin:
@$(SHELL) $(srcdir)/script/uninstallbin.sh $(INSTALLPERMS) $(DESTDIR)$(BASEDIR) $(DESTDIR)$(SBINDIR) $(DESTDIR)$(LIBDIR) $(DESTDIR)$(VARDIR) $(DESTDIR)$(SBIN_PROGS)
@$(SHELL) $(srcdir)/script/uninstallbin.sh $(INSTALLPERMS) $(DESTDIR)$(BASEDIR) $(DESTDIR)$(BINDIR) $(DESTDIR)$(LIBDIR) $(DESTDIR)$(VARDIR) $(DESTDIR)$(BIN_PROGS)
@$(SHELL) $(srcdir)/script/uninstallbin.sh $(INSTALLPERMS) $(DESTDIR)$(BASEDIR) $(DESTDIR)$(TORTUREDIR) $(DESTDIR)$(LIBDIR) $(DESTDIR)$(VARDIR) $(DESTDIR)$(TORTURE_PROGS)
uninstalllib:
@$(SHELL) $(srcdir)/script/uninstalllib.sh $(DESTDIR)$(LIBDIR) $(SHARED_LIBS)
@ -242,7 +252,7 @@ clean:: clean_pch
@echo Removing hostcc objects
@-find . -name '*.ho' -exec rm -f '{}' \;
@echo Removing binaries
@-rm -f $(BIN_PROGS) $(SBIN_PROGS) $(BINARIES)
@-rm -f $(BIN_PROGS) $(SBIN_PROGS) $(BINARIES) $(TORTURE_PROGS)
@echo Removing libraries
@-rm -f $(STATIC_LIBRARIES) $(SHARED_LIBRARIES)
@-rm -f bin/*.$(SHLIBEXT)*

View File

@ -105,8 +105,9 @@ cat >$CONFFILE<<EOF
server max protocol = SMB2
notify:inotify = false
ldb:nosync = true
torture:subunitdir = $SRCDIR/bin/torture
system:anonymous = true
system:anonymous = true
[tmp]
path = $TMPDIR

View File

@ -6,7 +6,8 @@ VERSION = 0.0.1
PUBLIC_HEADERS = torture.h
PUBLIC_PROTO_HEADER = proto.h
OBJ_FILES = \
torture.o
torture.o \
subunit.o
PUBLIC_DEPENDENCIES = \
LIBSAMBA-CONFIG \
LIBSAMBA-UTIL

View File

@ -7,7 +7,6 @@ PRIVATE_PROTO_HEADER = \
proto.h
OBJ_FILES = \
iconv.o \
../../lib/talloc/testsuite.o \
../../lib/replace/test/testsuite.o \
../../lib/replace/test/os2_delete.o \
../../lib/crypto/md4test.o \

View File

@ -52,12 +52,7 @@ NTSTATUS torture_local_init(void)
struct torture_suite *suite = torture_suite_create(
talloc_autofree_context(),
"LOCAL");
struct torture_suite *talloc_suite = torture_suite_create(
talloc_autofree_context(),
"TALLOC");
torture_local_talloc(talloc_suite);
torture_suite_add_suite(suite, talloc_suite);
torture_suite_add_simple_test(suite, "REPLACE", torture_local_replace);
torture_suite_add_simple_test(suite, "CRYPTO-SHA1",
torture_local_crypto_sha1);

View File

@ -27,6 +27,7 @@
#include "libcli/libcli.h"
#include "lib/ldb/include/ldb.h"
#include "lib/events/events.h"
#include "dynconfig.h"
#include "torture/torture.h"
#include "build.h"
@ -326,25 +327,27 @@ static void subunit_test_result (struct torture_context *context,
{
switch (res) {
case TORTURE_OK:
printf("success: %s\n", context->active_test->name);
printf("success: %s", context->active_test->name);
break;
case TORTURE_FAIL:
printf("failure: %s [ %s ]\n", context->active_test->name, reason);
printf("failure: %s", context->active_test->name);
break;
case TORTURE_TODO:
printf("todo: %s\n", context->active_test->name);
printf("todo: %s", context->active_test->name);
break;
case TORTURE_SKIP:
printf("skip: %s\n", context->active_test->name);
printf("skip: %s", context->active_test->name);
break;
}
if (reason)
printf(" [ %s ]", reason);
printf("\n");
}
static void subunit_comment (struct torture_context *test,
const char *comment)
{
/* FIXME Add # sign before each line */
printf("%s", comment);
fprintf(stderr, "%s", comment);
}
const static struct torture_ui_ops subunit_ui_ops = {
@ -438,6 +441,7 @@ const static struct torture_ui_ops quiet_ui_ops = {
char **argv_new;
poptContext pc;
static const char *target = "other";
const char **subunit_dir;
static const char *ui_ops_name = "simple";
enum {OPT_LOADFILE=1000,OPT_UNCLIST,OPT_TIMELIMIT,OPT_DNS,
OPT_DANGEROUS,OPT_SMB_PORTS,OPT_ASYNC};
@ -525,6 +529,17 @@ const static struct torture_ui_ops quiet_ui_ops = {
}
torture_init();
subunit_dir = lp_parm_string_list(-1, "torture", "subunitdir", ":");
if (subunit_dir == NULL)
torture_subunit_load_testsuites(dyn_TORTUREDIR);
else {
for (i = 0; subunit_dir[i]; i++)
torture_subunit_load_testsuites(subunit_dir[i]);
}
ldb_global_init();
if (torture_seed == 0) {

239
source4/torture/subunit.c Normal file
View File

@ -0,0 +1,239 @@
/*
Unix SMB/CIFS implementation.
Run subunit tests
Copyright (C) Jelmer Vernooij 2006
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "includes.h"
#include "system/dir.h"
#include "system/network.h"
#include "system/filesys.h"
#include "torture/ui.h"
#include "torture/proto.h"
NTSTATUS torture_register_subunit_testsuite(const char *path)
{
struct torture_suite *suite = talloc_zero(talloc_autofree_context(),
struct torture_suite);
suite->path = talloc_strdup(suite, path);
suite->name = talloc_strdup(suite, strrchr(path, '/')?strrchr(path, '/')+1:
path);
suite->description = talloc_asprintf(suite, "Subunit test %s", suite->name);
return torture_register_suite(suite);
}
int torture_subunit_load_testsuites(const char *directory)
{
DIR *dir;
struct dirent *entry;
char *filename;
int success = 0;
dir = opendir(directory);
if (dir == NULL)
return -1;
while((entry = readdir(dir))) {
if (ISDOT(entry->d_name) || ISDOTDOT(entry->d_name))
continue;
filename = talloc_asprintf(NULL, "%s/%s", directory, entry->d_name);
if (NT_STATUS_IS_OK(torture_register_subunit_testsuite(filename))) {
success++;
}
talloc_free(filename);
}
closedir(dir);
return success;
}
static pid_t piped_child(char* const command[], int *f_in)
{
pid_t pid;
int sock[2];
if (socketpair(PF_UNIX, SOCK_STREAM, AF_LOCAL, sock) == -1) {
DEBUG(0, ("socketpair: %s", strerror(errno)));
return -1;
}
*f_in = sock[0];
fcntl(sock[0], F_SETFL, O_NONBLOCK);
pid = fork();
if (pid == -1) {
DEBUG(0, ("fork: %s", strerror(errno)));
return -1;
}
if (pid == 0) {
close(0);
close(1);
close(2);
close(sock[0]);
dup2(sock[1], 0);
dup2(sock[1], 1);
execvp(command[0], command);
exit(-1);
}
close(sock[1]);
return pid;
}
enum subunit_field { SUBUNIT_TEST, SUBUNIT_SUCCESS, SUBUNIT_FAILURE,
SUBUNIT_SKIP };
static void run_subunit_message(struct torture_context *context,
enum subunit_field field,
const char *name,
const char *comment)
{
struct torture_test test;
ZERO_STRUCT(test);
test.name = name;
switch (field) {
case SUBUNIT_TEST:
torture_ui_test_start(context, NULL, &test);
break;
case SUBUNIT_FAILURE:
context->active_test = &test;
torture_ui_test_result(context, TORTURE_FAIL, comment);
context->active_test = NULL;
break;
case SUBUNIT_SUCCESS:
context->active_test = &test;
torture_ui_test_result(context, TORTURE_OK, comment);
context->active_test = NULL;
break;
case SUBUNIT_SKIP:
context->active_test = &test;
torture_ui_test_result(context, TORTURE_SKIP, comment);
context->active_test = NULL;
break;
}
}
bool torture_subunit_run_suite(struct torture_context *context,
struct torture_suite *suite)
{
static char *command[2];
int fd;
pid_t pid;
size_t size;
char *p, *q;
char *comment = NULL;
char *name = NULL;
enum subunit_field lastfield;
int status;
char buffer[4096];
size_t offset = 0;
command[0] = talloc_strdup(context, suite->path);
command[1] = NULL;
pid = piped_child(command, &fd);
if (pid == -1)
return false;
if (waitpid(pid, &status, 0) == -1) {
torture_comment(context, "waitpid(%d) failed\n", pid);
return false;
}
if (WEXITSTATUS(status) != 0) {
torture_comment(context, "failed with status %d\n", WEXITSTATUS(status));
return false;
}
while ((size = read(fd, buffer+offset, sizeof(buffer-offset) > 0))) {
char *eol;
buffer[offset+size] = '\0';
for (p = buffer; p; p = eol+1) {
eol = strchr(p, '\n');
if (eol == NULL)
break;
*eol = '\0';
if (comment != NULL && strcmp(p, "]") == 0) {
run_subunit_message(context, lastfield, name, comment);
talloc_free(name); name = NULL;
talloc_free(comment); comment = NULL;
} else if (comment != NULL) {
comment = talloc_append_string(context, comment, p);
} else {
q = strchr(p, ':');
if (q == NULL) {
torture_comment(context, "Invalid line `%s'\n", p);
continue;
}
*q = '\0';
if (!strcmp(p, "test")) {
lastfield = SUBUNIT_TEST;
} else if (!strcmp(p, "failure")) {
lastfield = SUBUNIT_FAILURE;
} else if (!strcmp(p, "success")) {
lastfield = SUBUNIT_SUCCESS;
} else if (!strcmp(p, "skip")) {
lastfield = SUBUNIT_SKIP;
} else {
torture_comment(context, "Invalid subunit field `%s'\n", p);
continue;
}
p = q+1;
name = talloc_strdup(context, p+1);
q = strrchr(p, '[');
if (q != NULL) {
*q = '\0';
comment = talloc_strdup(context, "");
} else {
run_subunit_message(context, lastfield, name, NULL);
talloc_free(name);
name = NULL;
}
}
}
offset += size-(p-buffer);
memcpy(buffer, p, offset);
}
if (name != NULL) {
torture_comment(context, "Interrupted during %s\n", name);
return false;
}
return true;
}

View File

@ -21,6 +21,7 @@
#include "includes.h"
#include "torture/ui.h"
#include "torture/torture.h"
#include "lib/util/dlinklist.h"
void torture_comment(struct torture_context *context,
@ -67,7 +68,7 @@ void _torture_skip_ext(struct torture_context *context,
struct torture_suite *torture_suite_create(TALLOC_CTX *ctx, const char *name)
{
struct torture_suite *suite = talloc(ctx, struct torture_suite);
struct torture_suite *suite = talloc_zero(ctx, struct torture_suite);
suite->name = talloc_strdup(suite, name);
suite->testcases = NULL;
@ -146,6 +147,9 @@ BOOL torture_run_suite(struct torture_context *context,
if (context->ui_ops->suite_start)
context->ui_ops->suite_start(context, suite);
if (suite->path)
torture_subunit_run_suite(context, suite);
for (tcase = suite->testcases; tcase; tcase = tcase->next) {
ret &= torture_run_tcase(context, tcase);
}
@ -162,6 +166,30 @@ BOOL torture_run_suite(struct torture_context *context,
return ret;
}
void torture_ui_test_start(struct torture_context *context,
struct torture_tcase *tcase,
struct torture_test *test)
{
if (context->ui_ops->test_start)
context->ui_ops->test_start(context, tcase, test);
}
void torture_ui_test_result(struct torture_context *context,
enum torture_result result,
const char *comment)
{
if (context->ui_ops->test_result)
context->ui_ops->test_result(context, result, comment);
switch (result) {
case TORTURE_SKIP: context->success++; break;
case TORTURE_FAIL: context->failed++; break;
case TORTURE_TODO: context->todo++; break;
case TORTURE_OK: context->success++; break;
}
}
static BOOL internal_torture_run_test(struct torture_context *context,
struct torture_tcase *tcase,
struct torture_test *test,
@ -182,8 +210,7 @@ static BOOL internal_torture_run_test(struct torture_context *context,
context->active_tcase = tcase;
context->active_test = test;
if (context->ui_ops->test_start)
context->ui_ops->test_start(context, tcase, test);
torture_ui_test_start(context, tcase, test);
context->last_reason = NULL;
context->last_result = TORTURE_OK;
@ -195,18 +222,7 @@ static BOOL internal_torture_run_test(struct torture_context *context,
context->last_result = TORTURE_FAIL;
}
if (context->ui_ops->test_result)
context->ui_ops->test_result(context,
context->last_result,
context->last_reason);
switch (context->last_result) {
case TORTURE_SKIP: context->success++; break;
case TORTURE_FAIL: context->failed++; break;
case TORTURE_TODO: context->todo++; break;
case TORTURE_OK: context->success++; break;
}
torture_ui_test_result(context, context->last_result, context->last_reason);
talloc_free(context->last_reason);

View File

@ -52,6 +52,14 @@ struct torture_ui_ops
enum torture_result, const char *reason);
};
void torture_ui_test_start(struct torture_context *context,
struct torture_tcase *tcase,
struct torture_test *test);
void torture_ui_test_result(struct torture_context *context,
enum torture_result result,
const char *comment);
/*
* Holds information about a specific run of the testsuite.
* The data in this structure should be considered private to
@ -121,6 +129,7 @@ struct torture_tcase {
struct torture_suite
{
const char *name;
const char *path; /* Used by subunit tests only */
const char *description;
struct torture_tcase *testcases;
struct torture_suite *children;