Implement syscall fault injection

Introduce new -e fault=EXPR syntax that can be used to specify a subset
of syscalls that are subject of syscall fault injection, an error code
that has to be injected, and a frequency of injection.

The expression specifying syscall fault injection has the following
format: SET[:error=ERRNO][:when=FIRST[+[STEP]]]
where only SET is a required part and all the rest is optional.

The method used to implement syscall fault injection is the following:
on entering syscall the syscall number is substituted by an invalid
syscall number -1, and on exiting syscall the error code returned by
the kernel is substituted with the error code specified in the fault
expression.

This implementaion is based on the prototype developed
by Nahim El Atmani as a part of his GSoC 2016 strace project.

* defs.h (struct fault_opts): New forward declaration.
(struct tcb): Add fault_vec field.
(TCB_FAULT_INJ, QUAL_FAULT): New macros.
* strace.1: Document -e fault expression syntax.
* strace.c (usage): Mention -e fault expression.
(droptcb): Deallocate fault_vec member.
* syscall.c (qual_fault, arch_set_scno, arch_set_error): New prototypes.
(qual_options): Add "fault" option.
(struct fault_opts): New structure.
(num_faults): New variable.
(fault_vec): New array.
(syscall_fault_injected, tcb_fault_opts, reallocate_fault,
find_errno_by_name, qual_syscall_ex, strip_prefix, parse_fault_token,
parse_fault_expression, qual_fault, inject_syscall_fault_entering,
update_syscall_fault_exiting): New functions.
(qual_syscall): Use qual_syscall_ex.
(qualify_one): Add argument: a pointer to struct fault_opts, all callers
changed.  Copy struct fault_opts from the pointer to fault_vec.
Use reallocate_fault.
(qualify_scno, qualify_syscall_class, qualify_syscall_name): Add
argument: a pointer to struct fault_opts.
(qualify): Use reallocate_fault.  Do not check "all" class for
QUAL_FAULT qualifier.
(lookup_class): Check for "all" class.
(trace_syscall_entering): Use inject_syscall_fault_entering.
(trace_syscall_exiting): Use update_syscall_fault_exiting.  Clear
TCB_FAULT_INJ flag along with TCB_INSYSCALL.  Print " (INJECTED)" suffix
when the syscall has been injected successfully.
[ARCH_REGS_FOR_GETREGSET && !HAVE_GETREGS_OLD]
(ptrace_setregset): New function.
(ptrace_setregset_or_setregs): Define to ptrace_setregset.
[ARCH_REGS_FOR_GETREGS && !HAVE_GETREGS_OLD]
(ptrace_setregs): New function.
(ptrace_setregset_or_setregs): Define to ptrace_setregs.
[ptrace_setregset_or_setregs] (set_regs): New function.
Include "set_scno.c" and "set_error.c"
* NEWS: Mention this enhancement.
This commit is contained in:
Дмитрий Левин 2016-11-16 17:26:58 +00:00
parent f1d0729f17
commit 9cb6e03d4f
5 changed files with 397 additions and 27 deletions

1
NEWS
View File

@ -1,6 +1,7 @@
Noteworthy changes in release ?.?? (????-??-??)
* Improvements
* Implemented syscall fault injection.
* Implemented decoding of DM_* ioctl commands.
* Implemented decoding of attr parameter of perf_event_open syscall.
* Implemented decoding of pkey_alloc, pkey_free, and pkey_mprotect syscalls.

5
defs.h
View File

@ -216,6 +216,8 @@ typedef struct ioctlent {
# define HAVE_STRUCT_TCB_EXT_ARG 0
#endif
struct fault_opts;
/* Trace Control Block */
struct tcb {
int flags; /* See below for TCB_ values */
@ -240,6 +242,7 @@ struct tcb {
void (*_free_priv_data)(void *); /* Callback for freeing priv_data */
const struct_sysent *s_ent; /* sysent[scno] or dummy struct for bad scno */
const struct_sysent *s_prev_ent; /* for "resuming interrupted SYSCALL" msg */
struct fault_opts *fault_vec[SUPPORTED_PERSONALITIES];
struct timeval stime; /* System time usage as of last process wait */
struct timeval dtime; /* Delta for system time usage */
struct timeval etime; /* Syscall entry time */
@ -276,6 +279,7 @@ struct tcb {
#define TCB_ATTACHED 0x08 /* We attached to it already */
#define TCB_REPRINT 0x10 /* We should reprint this syscall on exit */
#define TCB_FILTERED 0x20 /* This system call has been filtered out */
#define TCB_FAULT_INJ 0x40 /* A syscall fault has been injected */
/* qualifier flags */
#define QUAL_TRACE 0x001 /* this system call should be traced */
@ -285,6 +289,7 @@ struct tcb {
#define QUAL_SIGNAL 0x010 /* report events with this signal */
#define QUAL_READ 0x020 /* dump data read on this file descriptor */
#define QUAL_WRITE 0x040 /* dump data written to this file descriptor */
#define QUAL_FAULT 0x080 /* fail this system call on purpose */
typedef uint8_t qualbits_t;
#define DEFAULT_QUAL_FLAGS (QUAL_TRACE | QUAL_ABBREV | QUAL_VERBOSE)

View File

@ -355,8 +355,9 @@ is one of
.BR raw ,
.BR signal ,
.BR read ,
.BR write ,
or
.B write
.B fault
and
.I value
is a qualifier-dependent symbol or number. The default
@ -480,6 +481,83 @@ Note that this is independent from the normal tracing of the
.BR write (2)
system call which is controlled by the option
.BR -e "\ " trace = write .
.TP
\fB\-e\ fault\fR=\,\fIset\/\fR[:\fBerror\fR=\,\fIerrno\/\fR][:\fBwhen\fR=\,\fIexpr\/\fR]
Perform a syscall fault injection for the specified set of syscalls.
When a fault is injected into a syscall invocation, the syscall number
is replaced by -1 which corresponds to an invalid syscall.
If an error code is specified using a symbolic
.I errno
value like
.B ENOSYS
or a numeric value within 1..4095 range, this error code overrides
the default error code returned by the kernel, which is traditionally
.B ENOSYS
for invalid syscall numbers on most architectures.
Unless a :\fBwhen\fR=\,\fIexpr\fR subexpression is specified,
a fault is injected into every invocation of each syscall from the
.IR set .
The format of the subexpression is one of the following:
.RS
.IP "" 2
.I first
.RS 4
For every syscall from the
.IR set ,
perform a syscall fault injection for the syscall invocation number
.I first
only.
.RE
.IP "" 2
\fIfirst\/\fB+\fR
.RS 4
For every syscall from the
.IR set ,
perform syscall fault injections for the syscall invocation number
.I first
and all subsequent invocations.
.RE
.IP "" 2
\fIfirst\/\fB+\fIstep\fR
.RS 4
For every syscall from the
.IR set ,
perform syscall fault injections for syscall invocations number
.IR first ,
.IR first + step ,
.IR first + step + step ,
and so on.
.RE
.RE
.IP
For example, to fail each third and subsequent chdir syscalls with
.BR ENOENT ,
use
\fB\-e\ fault\fR=\,\fIchdir\/\fR:\fBerror\fR=\,\fIENOENT\/\fR:\fBwhen\fR=\,\fI3\/\fB+\fR.
The valid range for numbers
.I first
and
.I step
is 1..65535.
If a fault expression contains multiple
.BR error =
specifications, the last one takes precedence.
Likewise, if a fault expression contains multiple
.BR when =
specifications, the last one takes precedence.
Accounting of syscalls that are subject to fault injection
is done per syscall and per tracee.
Specification of syscall fault injection can be combined
with other syscall filtering options, for example,
\fB\-P \fI/dev/urandom \fB\-e fault\fR=\,\fIall\/\fR:\fBerror\fR=\,\fIENOENT\fR.
.TP
.BI "\-I " interruptible
When strace can be interrupted by signals (such as pressing ^C).

View File

@ -229,7 +229,7 @@ Statistics:\n\
\n\
Filtering:\n\
-e expr a qualifying expression: option=[!]all or option=[!]val1[,val2]...\n\
options: trace, abbrev, verbose, raw, signal, read, write\n\
options: trace, abbrev, verbose, raw, signal, read, write, fault\n\
-P path trace accesses to path\n\
\n\
Tracing:\n\
@ -785,6 +785,10 @@ droptcb(struct tcb *tcp)
if (tcp->pid == 0)
return;
int p;
for (p = 0; p < SUPPORTED_PERSONALITIES; ++p)
free(tcp->fault_vec[p]);
free_tcb_priv_data(tcp);
#ifdef USE_LIBUNWIND

332
syscall.c
View File

@ -360,6 +360,7 @@ update_personality(struct tcb *tcp, unsigned int personality)
#endif
static int qual_desc(const char *, unsigned int, int);
static int qual_fault(const char *, unsigned int, int);
static int qual_signal(const char *, unsigned int, int);
static int qual_syscall(const char *, unsigned int, int);
@ -386,6 +387,7 @@ static const struct qual_options {
{ QUAL_WRITE, "write", qual_desc, "descriptor" },
{ QUAL_WRITE, "writes", qual_desc, "descriptor" },
{ QUAL_WRITE, "w", qual_desc, "descriptor" },
{ QUAL_FAULT, "fault", qual_fault, "fault argument"},
{ 0, NULL, NULL, NULL },
};
@ -409,39 +411,65 @@ reallocate_qual(const unsigned int n)
num_quals = n;
}
struct fault_opts {
uint16_t first;
uint16_t step;
uint16_t err;
};
static unsigned int num_faults;
static struct fault_opts *fault_vec[SUPPORTED_PERSONALITIES];
static inline void
reallocate_fault(const unsigned int n)
{
reallocate_vec((void **) fault_vec, num_faults,
sizeof(struct fault_opts), n);
num_faults = n;
}
static void
qualify_one(const unsigned int n, unsigned int bitflag, const int not, const int pers)
qualify_one(const unsigned int n, unsigned int bitflag, const int not,
const int pers, const struct fault_opts *fopts)
{
int p;
if (num_quals <= n)
if (num_quals <= n) {
reallocate_qual(n + 1);
reallocate_fault(n + 1);
}
for (p = 0; p < SUPPORTED_PERSONALITIES; p++) {
if (pers == p || pers < 0) {
if (not)
qual_vec[p][n] &= ~bitflag;
else
else {
qual_vec[p][n] |= bitflag;
if (fopts)
memcpy(&fault_vec[p][n], fopts,
sizeof(*fopts));
}
}
}
}
static bool
qualify_scno(const char *const s, const unsigned int bitflag,
const int not)
const int not, const struct fault_opts *const fopts)
{
int i = string_to_uint_upto(s, MAX_NSYSCALLS - 1);
if (i < 0)
return false;
qualify_one(i, bitflag, not, -1);
qualify_one(i, bitflag, not, -1, fopts);
return true;
}
static int
lookup_class(const char *s)
{
if (strcmp(s, "all") == 0)
return 0;
if (strcmp(s, "file") == 0)
return TRACE_FILE;
if (strcmp(s, "ipc") == 0)
@ -461,7 +489,7 @@ lookup_class(const char *s)
static bool
qualify_syscall_class(const char *const s, const unsigned int bitflag,
const int not)
const int not, const struct fault_opts *const fopts)
{
unsigned int p;
const int n = lookup_class(s);
@ -475,7 +503,7 @@ qualify_syscall_class(const char *const s, const unsigned int bitflag,
for (i = 0; i < nsyscall_vec[p]; ++i) {
if (sysent_vec[p][i].sys_name
&& (sysent_vec[p][i].sys_flags & n) == n) {
qualify_one(i, bitflag, not, p);
qualify_one(i, bitflag, not, p, fopts);
}
}
}
@ -485,7 +513,7 @@ qualify_syscall_class(const char *const s, const unsigned int bitflag,
static bool
qualify_syscall_name(const char *const s, const unsigned int bitflag,
const int not)
const int not, const struct fault_opts *const fopts)
{
bool found = false;
unsigned int p;
@ -496,7 +524,7 @@ qualify_syscall_name(const char *const s, const unsigned int bitflag,
for (i = 0; i < nsyscall_vec[p]; ++i) {
if (sysent_vec[p][i].sys_name
&& strcmp(s, sysent_vec[p][i].sys_name) == 0) {
qualify_one(i, bitflag, not, p);
qualify_one(i, bitflag, not, p, fopts);
found = true;
}
}
@ -506,17 +534,146 @@ qualify_syscall_name(const char *const s, const unsigned int bitflag,
}
static int
qual_syscall(const char *s, const unsigned int bitflag, const int not)
qual_syscall_ex(const char *const s, const unsigned int bitflag,
const int not, const struct fault_opts *const fopts)
{
if (qualify_scno(s, bitflag, not)
|| qualify_syscall_class(s, bitflag, not)
|| qualify_syscall_name(s, bitflag, not)) {
if (qualify_scno(s, bitflag, not, fopts)
|| qualify_syscall_class(s, bitflag, not, fopts)
|| qualify_syscall_name(s, bitflag, not, fopts)) {
return 0;
}
return -1;
}
static int
qual_syscall(const char *const s, const unsigned int bitflag, const int not)
{
return qual_syscall_ex(s, bitflag, not, NULL);
}
/*
* Returns NULL if STR does not start with PREFIX,
* or a pointer to the first char in STR after PREFIX.
*/
static const char *
strip_prefix(const char *prefix, const char *str)
{
size_t len = strlen(prefix);
return (len > strlen(str) || memcmp(prefix, str, len))
? NULL : str + len;
}
static int
find_errno_by_name(const char *name)
{
unsigned int i;
for (i = 1; i < nerrnos; ++i) {
if (errnoent[i] && (strcmp(name, errnoent[i]) == 0))
return i;
}
return -1;
}
static bool
parse_fault_token(const char *const token, struct fault_opts *const fopts)
{
const char *val;
int intval;
if ((val = strip_prefix("when=", token))) {
/*
* == 1+1
* F == F+0
* F+ == F+1
* F+S
*/
char *end;
intval = string_to_uint_ex(val, &end, 0xffff, "+");
if (intval < 1)
return false;
fopts->first = intval;
if (*end) {
val = end + 1;
if (*val) {
/* F+S */
intval = string_to_uint_upto(val, 0xffff);
if (intval < 1)
return false;
fopts->step = intval;
} else {
/* F+ == F+1 */
fopts->step = 1;
}
} else {
/* F == F+0 */
fopts->step = 0;
}
} else if ((val = strip_prefix("error=", token))) {
intval = string_to_uint_upto(val, 4095);
if (intval < 0)
intval = find_errno_by_name(val);
if (intval < 1)
return false;
fopts->err = intval;
} else {
return false;
}
return true;
}
static const char *
parse_fault_expression(const char *const s, char **buf,
struct fault_opts *const fopts)
{
const char *name;
const char *token;
char *saveptr = NULL;
*buf = xstrdup(s);
name = strtok_r(*buf, ":", &saveptr);
if (!name || !*name)
goto parse_error;
while ((token = strtok_r(NULL, ":", &saveptr))) {
if (!parse_fault_token(token, fopts))
goto parse_error;
}
return name;
parse_error:
free(*buf);
return *buf = NULL;
}
static int
qual_fault(const char *const s, const unsigned int bitflag, const int not)
{
struct fault_opts opts = {
.first = 1,
.step = 1,
.err = 0
};
char *buf = NULL;
const char *name = parse_fault_expression(s, &buf, &opts);
if (!name)
return -1;
int rc = qual_syscall_ex(name, bitflag, not, &opts);
free(buf);
return rc;
}
static int
qual_signal(const char *s, const unsigned int bitflag, const int not)
{
@ -526,14 +683,14 @@ qual_signal(const char *s, const unsigned int bitflag, const int not)
int signo = string_to_uint_upto(s, 255);
if (signo < 0)
return -1;
qualify_one(signo, bitflag, not, -1);
qualify_one(signo, bitflag, not, -1, NULL);
return 0;
}
if (strncasecmp(s, "SIG", 3) == 0)
s += 3;
for (i = 0; i <= NSIG; i++) {
if (strcasecmp(s, signame(i) + 3) == 0) {
qualify_one(i, bitflag, not, -1);
qualify_one(i, bitflag, not, -1, NULL);
return 0;
}
}
@ -546,7 +703,7 @@ qual_desc(const char *s, const unsigned int bitflag, const int not)
int desc = string_to_uint_upto(s, 0x7fff);
if (desc < 0)
return -1;
qualify_one(desc, bitflag, not, -1);
qualify_one(desc, bitflag, not, -1, NULL);
return 0;
}
@ -559,8 +716,10 @@ qualify(const char *s)
int not;
unsigned int i;
if (num_quals == 0)
if (num_quals == 0) {
reallocate_qual(MIN_QUALS);
reallocate_fault(MIN_QUALS);
}
opt = &qual_options[0];
for (i = 0; (p = qual_options[i].option_name); i++) {
@ -580,14 +739,14 @@ qualify(const char *s)
not = 1 - not;
s = "all";
}
if (strcmp(s, "all") == 0) {
if (opt->bitflag != QUAL_FAULT && strcmp(s, "all") == 0) {
for (i = 0; i < num_quals; i++) {
qualify_one(i, opt->bitflag, not, -1);
qualify_one(i, opt->bitflag, not, -1, NULL);
}
return;
}
for (i = 0; i < num_quals; i++) {
qualify_one(i, opt->bitflag, !not, -1);
qualify_one(i, opt->bitflag, !not, -1, NULL);
}
copy = xstrdup(s);
for (p = strtok(copy, ","); p; p = strtok(NULL, ",")) {
@ -824,7 +983,62 @@ clear_regs(void)
static int get_syscall_args(struct tcb *);
static int get_syscall_result(struct tcb *);
static int arch_get_scno(struct tcb *tcp);
static int arch_set_scno(struct tcb *, long);
static void get_error(struct tcb *, const bool);
static int arch_set_error(struct tcb *);
static struct fault_opts *
tcb_fault_opts(struct tcb *tcp)
{
return (SCNO_IN_RANGE(tcp->scno) && tcp->fault_vec[current_personality])
? &tcp->fault_vec[current_personality][tcp->scno] : NULL;
}
static long
inject_syscall_fault_entering(struct tcb *tcp)
{
if (!tcp->fault_vec[current_personality]) {
tcp->fault_vec[current_personality] =
xreallocarray(NULL, num_faults,
sizeof(struct fault_opts));
memcpy(tcp->fault_vec[current_personality],
fault_vec[current_personality],
num_faults * sizeof(struct fault_opts));
}
struct fault_opts *opts = tcb_fault_opts(tcp);
if (opts->first == 0)
return 0;
--opts->first;
if (opts->first != 0)
return 0;
opts->first = opts->step;
if (!arch_set_scno(tcp, -1))
tcp->flags |= TCB_FAULT_INJ;
return 0;
}
static long
update_syscall_fault_exiting(struct tcb *tcp)
{
struct fault_opts *opts = tcb_fault_opts(tcp);
if (opts && opts->err && tcp->u_error != opts->err) {
unsigned long u_error = tcp->u_error;
tcp->u_error = opts->err;
if (arch_set_error(tcp))
tcp->u_error = u_error;
}
return 0;
}
static int
trace_syscall_entering(struct tcb *tcp)
@ -885,7 +1099,15 @@ trace_syscall_entering(struct tcb *tcp)
tcp->flags &= ~TCB_FILTERED;
if (cflag == CFLAG_ONLY_STATS || hide_log_until_execve) {
if (hide_log_until_execve) {
res = 0;
goto ret;
}
if (tcp->qual_flg & QUAL_FAULT)
inject_syscall_fault_entering(tcp);
if (cflag == CFLAG_ONLY_STATS) {
res = 0;
goto ret;
}
@ -914,6 +1136,12 @@ trace_syscall_entering(struct tcb *tcp)
return res;
}
static bool
syscall_fault_injected(struct tcb *tcp)
{
return tcp->flags & TCB_FAULT_INJ;
}
static int
trace_syscall_exiting(struct tcb *tcp)
{
@ -941,6 +1169,9 @@ trace_syscall_exiting(struct tcb *tcp)
if (filtered(tcp) || hide_log_until_execve)
goto ret;
if (syserror(tcp) && syscall_fault_injected(tcp))
update_syscall_fault_exiting(tcp);
if (cflag) {
count_syscall(tcp, &tv);
if (cflag == CFLAG_ONLY_STATS) {
@ -971,7 +1202,7 @@ trace_syscall_exiting(struct tcb *tcp)
tabto();
tprints("= ? <unavailable>\n");
line_ended();
tcp->flags &= ~TCB_INSYSCALL;
tcp->flags &= ~(TCB_INSYSCALL | TCB_FAULT_INJ);
tcp->sys_func_rval = 0;
free_tcb_priv_data(tcp);
return res;
@ -1001,11 +1232,15 @@ trace_syscall_exiting(struct tcb *tcp)
tprints(") ");
tabto();
u_error = tcp->u_error;
if (tcp->qual_flg & QUAL_RAW) {
if (u_error)
if (u_error) {
tprintf("= -1 (errno %lu)", u_error);
else
if (syscall_fault_injected(tcp))
tprints(" (INJECTED)");
} else {
tprintf("= %#lx", tcp->u_rval);
}
}
else if (!(sys_res & RVAL_NONE) && u_error) {
switch (u_error) {
@ -1072,6 +1307,8 @@ trace_syscall_exiting(struct tcb *tcp)
u_error, strerror(u_error));
break;
}
if (syscall_fault_injected(tcp))
tprintf(" (INJECTED)");
if ((sys_res & RVAL_STR) && tcp->auxstr)
tprintf(" (%s)", tcp->auxstr);
}
@ -1154,7 +1391,7 @@ trace_syscall_exiting(struct tcb *tcp)
#endif
ret:
tcp->flags &= ~TCB_INSYSCALL;
tcp->flags &= ~(TCB_INSYSCALL | TCB_FAULT_INJ);
tcp->sys_func_rval = 0;
free_tcb_priv_data(tcp);
return 0;
@ -1257,6 +1494,7 @@ print_pc(struct tcb *tcp)
#include "getregs_old.h"
#undef ptrace_getregset_or_getregs
#undef ptrace_setregset_or_setregs
#ifdef ARCH_REGS_FOR_GETREGSET
# define ptrace_getregset_or_getregs ptrace_getregset
@ -1279,6 +1517,26 @@ ptrace_getregset(pid_t pid)
# endif
}
# ifndef HAVE_GETREGS_OLD
# define ptrace_setregset_or_setregs ptrace_setregset
static int
ptrace_setregset(pid_t pid)
{
# ifdef ARCH_IOVEC_FOR_GETREGSET
/* variable iovec */
return ptrace(PTRACE_SETREGSET, pid, NT_PRSTATUS,
&ARCH_IOVEC_FOR_GETREGSET);
# else
/* constant iovec */
static struct iovec io = {
.iov_base = &ARCH_REGS_FOR_GETREGSET,
.iov_len = sizeof(ARCH_REGS_FOR_GETREGSET)
};
return ptrace(PTRACE_SETREGSET, pid, NT_PRSTATUS, &io);
# endif
}
# endif /* !HAVE_GETREGS_OLD */
#elif defined ARCH_REGS_FOR_GETREGS
# define ptrace_getregset_or_getregs ptrace_getregs
@ -1293,6 +1551,20 @@ ptrace_getregs(pid_t pid)
# endif
}
# ifndef HAVE_GETREGS_OLD
# define ptrace_setregset_or_setregs ptrace_setregs
static int
ptrace_setregs(pid_t pid)
{
# if defined SPARC || defined SPARC64
/* SPARC systems have the meaning of data and addr reversed */
return ptrace(PTRACE_SETREGS, pid, (void *) &ARCH_REGS_FOR_GETREGS, 0);
# else
return ptrace(PTRACE_SETREGS, pid, NULL, &ARCH_REGS_FOR_GETREGS);
# endif
}
# endif /* !HAVE_GETREGS_OLD */
#endif /* ARCH_REGS_FOR_GETREGSET || ARCH_REGS_FOR_GETREGS */
void
@ -1335,6 +1607,14 @@ get_regs(pid_t pid)
#endif /* !ptrace_getregset_or_getregs */
}
#ifdef ptrace_setregset_or_setregs
static int
set_regs(pid_t pid)
{
return ptrace_setregset_or_setregs(pid);
}
#endif /* ptrace_setregset_or_setregs */
struct sysent_buf {
struct tcb *tcp;
struct_sysent ent;
@ -1413,11 +1693,13 @@ get_syscall_result(struct tcb *tcp)
}
#include "get_scno.c"
#include "set_scno.c"
#include "get_syscall_args.c"
#ifdef USE_GET_SYSCALL_RESULT_REGS
# include "get_syscall_result.c"
#endif
#include "get_error.c"
#include "set_error.c"
#ifdef HAVE_GETREGS_OLD
# include "getregs_old.c"
#endif