From df99d9e2739eb8e5448bc9cfdf3c469d396dd3e3 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 17 Feb 2016 14:32:03 +1100 Subject: [PATCH] ctdb-cluster-mutex: Factor out cluster mutex code Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs --- ctdb/server/ctdb_cluster_mutex.c | 266 +++++++++++++++++++++++++++++++ ctdb/server/ctdb_cluster_mutex.h | 48 ++++++ ctdb/server/ctdb_recover.c | 246 +--------------------------- ctdb/tests/src/ctdbd_test.c | 1 + ctdb/wscript | 1 + 5 files changed, 318 insertions(+), 244 deletions(-) create mode 100644 ctdb/server/ctdb_cluster_mutex.c create mode 100644 ctdb/server/ctdb_cluster_mutex.h diff --git a/ctdb/server/ctdb_cluster_mutex.c b/ctdb/server/ctdb_cluster_mutex.c new file mode 100644 index 00000000000..12950c443d0 --- /dev/null +++ b/ctdb/server/ctdb_cluster_mutex.c @@ -0,0 +1,266 @@ +/* + CTDB cluster mutex handling + + Copyright (C) Andrew Tridgell 2007 + Copyright (C) Ronnie Sahlberg 2007 + Copyright (C) Martin Schwenke 2016 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . +*/ + +#include + +#include "replace.h" +#include "system/network.h" + +#include "lib/util/debug.h" +#include "lib/util/time.h" +#include "lib/util/strv.h" +#include "lib/util/strv_util.h" + +#include "ctdb_private.h" +#include "common/common.h" +#include "common/logging.h" +#include "common/system.h" + +#include "ctdb_cluster_mutex.h" + +struct ctdb_cluster_mutex_handle { + struct ctdb_context *ctdb; + cluster_mutex_handler_t handler; + void *private_data; + int fd[2]; + struct tevent_timer *te; + struct tevent_fd *fde; + pid_t child; + struct timeval start_time; +}; + +void ctdb_cluster_mutex_set_handler(struct ctdb_cluster_mutex_handle *h, + cluster_mutex_handler_t handler, + void *private_data) +{ + h->handler = handler; + h->private_data = private_data; +} + +static void cluster_mutex_timeout(struct tevent_context *ev, + struct tevent_timer *te, + struct timeval t, void *private_data) +{ + struct ctdb_cluster_mutex_handle *h = + talloc_get_type(private_data, struct ctdb_cluster_mutex_handle); + double latency = timeval_elapsed(&h->start_time); + + if (h->handler != NULL) { + h->handler(h->ctdb, '2', latency, h, h->private_data); + } +} + + +/* When the handle is freed it causes any child holding the mutex to + * be killed, thus freeing the mutex */ +static int cluster_mutex_destructor(struct ctdb_cluster_mutex_handle *h) +{ + if (h->fd[0] != -1) { + h->fd[0] = -1; + } + ctdb_kill(h->ctdb, h->child, SIGTERM); + return 0; +} + +/* this is called when the client process has completed ctdb_recovery_lock() + and has written data back to us through the pipe. +*/ +static void cluster_mutex_handler(struct tevent_context *ev, + struct tevent_fd *fde, + uint16_t flags, void *private_data) +{ + struct ctdb_cluster_mutex_handle *h= + talloc_get_type(private_data, struct ctdb_cluster_mutex_handle); + double latency = timeval_elapsed(&h->start_time); + char c = '0'; + int ret; + + /* Got response from child process so abort timeout */ + TALLOC_FREE(h->te); + + ret = sys_read(h->fd[0], &c, 1); + + /* If the child wrote status then just pass it to the handler. + * If no status was written then this is an unexpected error + * so pass generic error code to handler. */ + if (h->handler != NULL) { + h->handler(h->ctdb, ret == 1 ? c : '3', latency, + h, h->private_data); + } +} + +static char cluster_mutex_helper[PATH_MAX+1] = ""; + +static bool cluster_mutex_helper_args(TALLOC_CTX *mem_ctx, + const char *argstring, char ***argv) +{ + int nargs, i, ret, n; + bool is_command = false; + char **args = NULL; + char *strv = NULL; + char *t = NULL; + + if (argstring != NULL && argstring[0] == '!') { + /* This is actually a full command */ + is_command = true; + t = discard_const(&argstring[1]); + } else { + is_command = false; + t = discard_const(argstring); + } + + ret = strv_split(mem_ctx, &strv, t, " \t"); + if (ret != 0) { + DEBUG(DEBUG_ERR, + ("Unable to parse mutex helper string \"%s\" (%s)\n", + argstring, strerror(ret))); + return false; + } + n = strv_count(strv); + + args = talloc_array(mem_ctx, char *, n + (is_command ? 1 : 2)); + + if (args == NULL) { + DEBUG(DEBUG_ERR,(__location__ " out of memory\n")); + return false; + } + + nargs = 0; + + if (! is_command) { + if (!ctdb_set_helper("cluster mutex helper", + cluster_mutex_helper, + sizeof(cluster_mutex_helper), + "CTDB_CLUSTER_MUTEX_HELPER", + CTDB_HELPER_BINDIR, + "ctdb_mutex_fcntl_helper")) { + DEBUG(DEBUG_ERR,("ctdb exiting with error: %s\n", + __location__ + " Unable to set cluster mutex helper\n")); + exit(1); + } + + args[nargs++] = cluster_mutex_helper; + } + + t = NULL; + for (i = 0; i < n; i++) { + /* Don't copy, just keep cmd_args around */ + t = strv_next(strv, t); + args[nargs++] = t; + } + + /* Make sure last argument is NULL */ + args[nargs] = NULL; + + *argv = args; + return true; +} + +struct ctdb_cluster_mutex_handle * +ctdb_cluster_mutex(struct ctdb_context *ctdb, + const char *argstring, + int timeout) +{ + struct ctdb_cluster_mutex_handle *h; + char **args; + int ret; + + h = talloc(ctdb, struct ctdb_cluster_mutex_handle); + if (h == NULL) { + DEBUG(DEBUG_ERR, (__location__ " out of memory\n")); + return NULL; + } + + h->start_time = timeval_current(); + h->fd[0] = -1; + h->fd[1] = -1; + + ret = pipe(h->fd); + if (ret != 0) { + talloc_free(h); + DEBUG(DEBUG_ERR, (__location__ " Failed to open pipe\n")); + return NULL; + } + set_close_on_exec(h->fd[0]); + + /* Create arguments for lock helper */ + if (!cluster_mutex_helper_args(h, argstring, &args)) { + close(h->fd[0]); + close(h->fd[1]); + talloc_free(h); + return NULL; + } + + h->child = ctdb_fork(ctdb); + if (h->child == (pid_t)-1) { + close(h->fd[0]); + close(h->fd[1]); + talloc_free(h); + return NULL; + } + + if (h->child == 0) { + /* Make stdout point to the pipe */ + close(STDOUT_FILENO); + dup2(h->fd[1], STDOUT_FILENO); + close(h->fd[1]); + + execv(args[0], args); + + /* Only happens on error */ + DEBUG(DEBUG_ERR, (__location__ "execv() failed\n")); + _exit(1); + } + + /* Parent */ + + DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d\n", h->fd[0])); + set_close_on_exec(h->fd[0]); + + close(h->fd[1]); + h->fd[1] = -1; + + talloc_set_destructor(h, cluster_mutex_destructor); + + if (timeout != 0) { + h->te = tevent_add_timer(ctdb->ev, h, + timeval_current_ofs(timeout, 0), + cluster_mutex_timeout, h); + } else { + h->te = NULL; + } + + h->fde = tevent_add_fd(ctdb->ev, h, h->fd[0], TEVENT_FD_READ, + cluster_mutex_handler, (void *)h); + + if (h->fde == NULL) { + talloc_free(h); + return NULL; + } + tevent_fd_set_auto_close(h->fde); + + h->ctdb = ctdb; + h->handler = NULL; + h->private_data = NULL; + + return h; +} diff --git a/ctdb/server/ctdb_cluster_mutex.h b/ctdb/server/ctdb_cluster_mutex.h new file mode 100644 index 00000000000..9131eb31d0f --- /dev/null +++ b/ctdb/server/ctdb_cluster_mutex.h @@ -0,0 +1,48 @@ +/* + CTDB cluster mutex handling + + Copyright (C) Andrew Tridgell 2007 + Copyright (C) Ronnie Sahlberg 2007 + Copyright (C) Martin Schwenke 2016 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . +*/ + +#ifndef __CTDB_CLUSTER_MUTEX_H__ +#define __CTDB_CLUSTER_MUTEX_H__ + +#include "replace.h" +#include "system/network.h" + +#include "ctdb_private.h" + +struct ctdb_cluster_mutex_handle; + +typedef void (*cluster_mutex_handler_t) ( + struct ctdb_context *ctdb, + char status, + double latency, + struct ctdb_cluster_mutex_handle *h, + void *private_data); + +void ctdb_cluster_mutex_set_handler(struct ctdb_cluster_mutex_handle *h, + cluster_mutex_handler_t handler, + void *private_data); + +struct ctdb_cluster_mutex_handle * +ctdb_cluster_mutex(struct ctdb_context *ctdb, + const char *argstring, + int timeout); + +#endif /* __CTDB_IPALLOC_H__ */ diff --git a/ctdb/server/ctdb_recover.c b/ctdb/server/ctdb_recover.c index b8f71ff5377..fe7e6e9cb35 100644 --- a/ctdb/server/ctdb_recover.c +++ b/ctdb/server/ctdb_recover.c @@ -32,8 +32,6 @@ #include "lib/util/debug.h" #include "lib/util/time.h" #include "lib/util/util_process.h" -#include "lib/util/strv.h" -#include "lib/util/strv_util.h" #include "ctdb_private.h" #include "ctdb_client.h" @@ -42,6 +40,8 @@ #include "common/common.h" #include "common/logging.h" +#include "ctdb_cluster_mutex.h" + int ctdb_control_getvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata) { @@ -749,33 +749,6 @@ int32_t ctdb_control_db_push_confirm(struct ctdb_context *ctdb, return 0; } -struct ctdb_cluster_mutex_handle; -typedef void (*cluster_mutex_handler_t) ( - struct ctdb_context *ctdb, - char status, - double latency, - struct ctdb_cluster_mutex_handle *h, - void *private_data); - -struct ctdb_cluster_mutex_handle { - struct ctdb_context *ctdb; - cluster_mutex_handler_t handler; - void *private_data; - int fd[2]; - struct tevent_timer *te; - struct tevent_fd *fde; - pid_t child; - struct timeval start_time; -}; - -static void ctdb_cluster_mutex_set_handler(struct ctdb_cluster_mutex_handle *h, - cluster_mutex_handler_t handler, - void *private_data) -{ - h->handler = handler; - h->private_data = private_data; -} - static void set_recmode_handler(struct ctdb_context *ctdb, char status, double latency, @@ -839,62 +812,6 @@ static void set_recmode_handler(struct ctdb_context *ctdb, talloc_free(h); } -/* - called if our set_recmode child times out. this would happen if - ctdb_recovery_lock() would block. - */ -static void cluster_mutex_timeout(struct tevent_context *ev, - struct tevent_timer *te, - struct timeval t, void *private_data) -{ - struct ctdb_cluster_mutex_handle *h = - talloc_get_type(private_data, struct ctdb_cluster_mutex_handle); - double latency = timeval_elapsed(&h->start_time); - - if (h->handler != NULL) { - h->handler(h->ctdb, '2', latency, h, h->private_data); - } -} - - -/* When the handle is freed it causes any child holding the mutex to - * be killed, thus freeing the mutex */ -static int cluster_mutex_destructor(struct ctdb_cluster_mutex_handle *h) -{ - if (h->fd[0] != -1) { - h->fd[0] = -1; - } - ctdb_kill(h->ctdb, h->child, SIGTERM); - return 0; -} - -/* this is called when the client process has completed ctdb_recovery_lock() - and has written data back to us through the pipe. -*/ -static void cluster_mutex_handler(struct tevent_context *ev, - struct tevent_fd *fde, - uint16_t flags, void *private_data) -{ - struct ctdb_cluster_mutex_handle *h= - talloc_get_type(private_data, struct ctdb_cluster_mutex_handle); - double latency = timeval_elapsed(&h->start_time); - char c = '0'; - int ret; - - /* Got response from child process so abort timeout */ - TALLOC_FREE(h->te); - - ret = sys_read(h->fd[0], &c, 1); - - /* If the child wrote status then just pass it to the handler. - * If no status was written then this is an unexpected error - * so pass generic error code to handler. */ - if (h->handler != NULL) { - h->handler(h->ctdb, ret == 1 ? c : '3', latency, - h, h->private_data); - } -} - static void ctdb_drop_all_ips_event(struct tevent_context *ev, struct tevent_timer *te, struct timeval t, void *private_data) @@ -908,165 +825,6 @@ ctdb_drop_all_ips_event(struct tevent_context *ev, struct tevent_timer *te, ctdb_release_all_ips(ctdb); } -static char cluster_mutex_helper[PATH_MAX+1] = ""; - -static bool cluster_mutex_helper_args(TALLOC_CTX *mem_ctx, - const char *argstring, char ***argv) -{ - int nargs, i, ret, n; - bool is_command = false; - char **args = NULL; - char *strv = NULL; - char *t = NULL; - - if (argstring != NULL && argstring[0] == '!') { - /* This is actually a full command */ - is_command = true; - t = discard_const(&argstring[1]); - } else { - is_command = false; - t = discard_const(argstring); - } - - ret = strv_split(mem_ctx, &strv, t, " \t"); - if (ret != 0) { - DEBUG(DEBUG_ERR, - ("Unable to parse mutex helper string \"%s\" (%s)\n", - argstring, strerror(ret))); - return false; - } - n = strv_count(strv); - - args = talloc_array(mem_ctx, char *, n + (is_command ? 1 : 2)); - - if (args == NULL) { - DEBUG(DEBUG_ERR,(__location__ " out of memory\n")); - return false; - } - - nargs = 0; - - if (! is_command) { - if (!ctdb_set_helper("cluster mutex helper", - cluster_mutex_helper, - sizeof(cluster_mutex_helper), - "CTDB_CLUSTER_MUTEX_HELPER", - CTDB_HELPER_BINDIR, - "ctdb_mutex_fcntl_helper")) { - DEBUG(DEBUG_ERR,("ctdb exiting with error: %s\n", - __location__ - " Unable to set cluster mutex helper\n")); - exit(1); - } - - args[nargs++] = cluster_mutex_helper; - } - - t = NULL; - for (i = 0; i < n; i++) { - /* Don't copy, just keep cmd_args around */ - t = strv_next(strv, t); - args[nargs++] = t; - } - - /* Make sure last argument is NULL */ - args[nargs] = NULL; - - *argv = args; - return true; -} - -static struct ctdb_cluster_mutex_handle * -ctdb_cluster_mutex(struct ctdb_context *ctdb, - const char *argstring, - int timeout) -{ - struct ctdb_cluster_mutex_handle *h; - char **args; - int ret; - - h = talloc(ctdb, struct ctdb_cluster_mutex_handle); - if (h == NULL) { - DEBUG(DEBUG_ERR, (__location__ " out of memory\n")); - return NULL; - } - - h->start_time = timeval_current(); - h->fd[0] = -1; - h->fd[1] = -1; - - ret = pipe(h->fd); - if (ret != 0) { - talloc_free(h); - DEBUG(DEBUG_ERR, (__location__ " Failed to open pipe\n")); - return NULL; - } - set_close_on_exec(h->fd[0]); - - /* Create arguments for lock helper */ - if (!cluster_mutex_helper_args(h, argstring, &args)) { - close(h->fd[0]); - close(h->fd[1]); - talloc_free(h); - return NULL; - } - - h->child = ctdb_fork(ctdb); - if (h->child == (pid_t)-1) { - close(h->fd[0]); - close(h->fd[1]); - talloc_free(h); - return NULL; - } - - if (h->child == 0) { - /* Make stdout point to the pipe */ - close(STDOUT_FILENO); - dup2(h->fd[1], STDOUT_FILENO); - close(h->fd[1]); - - execv(args[0], args); - - /* Only happens on error */ - DEBUG(DEBUG_ERR, (__location__ "execv() failed\n")); - _exit(1); - } - - /* Parent */ - - DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d\n", h->fd[0])); - set_close_on_exec(h->fd[0]); - - close(h->fd[1]); - h->fd[1] = -1; - - talloc_set_destructor(h, cluster_mutex_destructor); - - if (timeout != 0) { - h->te = tevent_add_timer(ctdb->ev, h, - timeval_current_ofs(timeout, 0), - cluster_mutex_timeout, h); - } else { - h->te = NULL; - } - - h->fde = tevent_add_fd(ctdb->ev, h, h->fd[0], TEVENT_FD_READ, - cluster_mutex_handler, (void *)h); - - if (h->fde == NULL) { - talloc_free(h); - return NULL; - } - tevent_fd_set_auto_close(h->fde); - - h->ctdb = ctdb; - h->handler = NULL; - h->private_data = NULL; - - return h; -} - - /* * Set up an event to drop all public ips if we remain in recovery for too * long diff --git a/ctdb/tests/src/ctdbd_test.c b/ctdb/tests/src/ctdbd_test.c index 77fac08a6ad..7d66712200c 100644 --- a/ctdb/tests/src/ctdbd_test.c +++ b/ctdb/tests/src/ctdbd_test.c @@ -64,6 +64,7 @@ bool fast_start; #include "server/ctdb_takeover.c" #include "server/ctdb_persistent.c" #include "server/ctdb_keepalive.c" +#include "server/ctdb_cluster_mutex.c" #include "server/ctdb_logging.c" #include "server/ctdb_logging_syslog.c" #include "server/ctdb_logging_file.c" diff --git a/ctdb/wscript b/ctdb/wscript index 6a50a3966fa..c51d963e7c6 100755 --- a/ctdb/wscript +++ b/ctdb/wscript @@ -394,6 +394,7 @@ def build(bld): ctdb_traverse.c eventscript.c ctdb_takeover.c ctdb_persistent.c ctdb_keepalive.c + ctdb_cluster_mutex.c ctdb_logging.c ctdb_logging_syslog.c ctdb_logging_file.c