diff --git a/WHATS_NEW b/WHATS_NEW index 2b54f37fe..18b511d49 100644 --- a/WHATS_NEW +++ b/WHATS_NEW @@ -1,5 +1,6 @@ Version 2.02.87 - =============================== + Remove obsoleted GULM clvmd cluster locking support. Suppress low-level locking errors and warnings while using --sysinit. Remove unused inconsistent_seqno variable in _vg_read(). Remove meaningless const type qualifiers on cast type. diff --git a/configure.in b/configure.in index 29120fc20..7783f5ce6 100644 --- a/configure.in +++ b/configure.in @@ -423,7 +423,6 @@ AC_MSG_CHECKING(whether to build cluster LVM daemon) AC_ARG_WITH(clvmd, [ --with-clvmd=TYPE build cluster LVM Daemon The following cluster manager combinations are valid: - * cman,gulm (RHEL4 or equivalent) * cman (RHEL5 or equivalent) * cman,corosync,openais (or selection of them) * singlenode (localhost only) @@ -449,14 +448,12 @@ fi dnl -- Express clvmd init script Required-Start / Required-Stop CLVMD_CMANAGERS="" dnl -- On RHEL4/RHEL5, qdiskd is started from a separate init script. -dnl -- Enable if we are build for either cman or gulm. +dnl -- Enable if we are build for cman. CLVMD_NEEDS_QDISKD=no dnl -- define build types if [[ `expr x"$CLVMD" : '.*gulm.*'` != 0 ]]; then - BUILDGULM=yes - CLVMD_CMANAGERS="$CLVMD_CMANAGERS lock_gulmd" - CLVMD_NEEDS_QDISKD=yes + AC_MSG_ERROR([Since version 2.02.87 GULM locking is no longer supported.]); fi if [[ `expr x"$CLVMD" : '.*cman.*'` != 0 ]]; then BUILDCMAN=yes @@ -475,14 +472,6 @@ if test x$CLVMD_NEEDS_QDISKD != xno; then CLVMD_CMANAGERS="$CLVMD_CMANAGERS qdiskd" fi -dnl -- sanity check around user selection -if test x$BUILDGULM = xyes; then - if test x$BUILDCOROSYNC = xyes || \ - test x$BUILDOPENAIS = xyes; then - AC_MSG_ERROR([requested clvmd configuration is not valid]) - fi -fi - dnl -- define a soft bailout if we are autodetecting soft_bailout() { NOTFOUND=1 @@ -494,10 +483,9 @@ hard_bailout() { dnl -- if clvmd=all then set soft_bailout (we don't want to error) dnl -- and set all builds to yes. We need to do this here -dnl -- to skip the gulm + openais|corosync sanity check above. +dnl -- to skip the openais|corosync sanity check above. if test x$CLVMD = xall; then bailout=soft_bailout - BUILDGULM=yes BUILDCMAN=yes BUILDCOROSYNC=yes BUILDOPENAIS=yes @@ -519,28 +507,6 @@ check_lib_no_libs() { LIBS=$ac_check_lib_save_LIBS } -dnl -- Look for gulm libraries if required. -if test x$BUILDGULM = xyes; then - PKG_CHECK_MODULES(CCS, libccs, [HAVE_CCS=yes], - [NOTFOUND=0 - AC_CHECK_HEADERS(ccs.h,,$bailout) - check_lib_no_libs ccs ccs_connect - if test $NOTFOUND = 0; then - AC_MSG_RESULT([no pkg for libccs, using -lccs]) - CCS_LIBS="-lccs" - HAVE_CCS=yes - fi]) - PKG_CHECK_MODULES(GULM, libgulm, [HAVE_GULM=yes], - [NOTFOUND=0 - AC_CHECK_HEADERS(libgulm.h,,$bailout) - check_lib_no_libs gulm lg_core_login - if test $NOTFOUND = 0; then - AC_MSG_RESULT([no pkg for libgulm, using -lgulm]) - GULM_LIBS="-lgulm" - HAVE_GULM=yes - fi]) -fi - dnl -- Look for cman libraries if required. if test x$BUILDCMAN = xyes; then PKG_CHECK_MODULES(CMAN, libcman, [HAVE_CMAN=yes], @@ -633,13 +599,6 @@ if test x$CLVMD = xall; then CLVMD=none CLVMD_CMANAGERS="" CLVMD_NEEDS_QDISKD=no - if test x$HAVE_CCS = xyes && \ - test x$HAVE_GULM = xyes; then - AC_MSG_RESULT([Enabling clvmd gulm cluster manager]) - CLVMD="$CLVMD,gulm" - CLVMD_CMANAGERS="$CLVMD_CMANAGERS lock_gulmd" - CLVMD_NEEDS_QDISKD=yes - fi if test x$HAVE_CMAN = xyes && \ test x$HAVE_DLM = xyes; then AC_MSG_RESULT([Enabling clvmd cman cluster manager]) @@ -1354,8 +1313,6 @@ AC_SUBST(DM_IOCTLS) AC_SUBST(DM_LIB_VERSION) AC_SUBST(DM_LIB_PATCHLEVEL) AC_SUBST(FSADM) -AC_SUBST(GULM_CFLAGS) -AC_SUBST(GULM_LIBS) AC_SUBST(HAVE_LIBDL) AC_SUBST(HAVE_REALTIME) AC_SUBST(INTL) diff --git a/daemons/clvmd/Makefile.in b/daemons/clvmd/Makefile.in index ee19e6ceb..91fd2731f 100644 --- a/daemons/clvmd/Makefile.in +++ b/daemons/clvmd/Makefile.in @@ -25,8 +25,6 @@ CPG_LIBS = @CPG_LIBS@ CPG_CFLAGS = @CPG_CFLAGS@ DLM_LIBS = @DLM_LIBS@ DLM_CFLAGS = @DLM_CFLAGS@ -GULM_LIBS = @GULM_LIBS@ -GULM_CFLAGS = @GULM_CFLAGS@ QUORUM_LIBS = @QUORUM_LIBS@ QUORUM_CFLAGS = @QUORUM_CFLAGS@ SALCK_LIBS = @SALCK_LIBS@ @@ -42,13 +40,6 @@ ifeq ("@DEBUG@", "yes") DEFS += -DDEBUG endif -ifneq (,$(findstring gulm,, "@CLVMD@,")) - SOURCES += clvmd-gulm.c tcp-comms.c - LMLIBS += $(CCS_LIBS) $(GULM_LIBS) - CFLAGS += $(CCS_CFLAGS) $(GULM_CFLAGS) - DEFS += -DUSE_GULM -endif - ifneq (,$(findstring cman,, "@CLVMD@,")) SOURCES += clvmd-cman.c LMLIBS += $(CMAN_LIBS) $(CONFDB_LIBS) $(DLM_LIBS) @@ -76,7 +67,6 @@ ifneq (,$(findstring singlenode,, "@CLVMD@,")) endif ifeq ($(MAKECMDGOALS),distclean) - SOURCES += clvmd-gulm.c tcp-comms.c SOURCES += clvmd-cman.c SOURCES += clvmd-openais.c SOURCES += clvmd-corosync.c diff --git a/daemons/clvmd/clvmd-comms.h b/daemons/clvmd/clvmd-comms.h index fbcfe8b54..500bd5752 100644 --- a/daemons/clvmd/clvmd-comms.h +++ b/daemons/clvmd/clvmd-comms.h @@ -54,13 +54,6 @@ struct cluster_ops { }; -#ifdef USE_GULM -# include "tcp-comms.h" -struct cluster_ops *init_gulm_cluster(void); -#define MAX_CSID_LEN GULM_MAX_CSID_LEN -#define MAX_CLUSTER_MEMBER_NAME_LEN GULM_MAX_CLUSTER_MEMBER_NAME_LEN -#endif - #ifdef USE_CMAN # include # include "libcman.h" diff --git a/daemons/clvmd/clvmd-gulm.c b/daemons/clvmd/clvmd-gulm.c deleted file mode 100644 index 356100412..000000000 --- a/daemons/clvmd/clvmd-gulm.c +++ /dev/null @@ -1,1010 +0,0 @@ -/* - * Copyright (C) 2002-2003 Sistina Software, Inc. All rights reserved. - * Copyright (C) 2004-2009 Red Hat, Inc. All rights reserved. - * - * This file is part of LVM2. - * - * This copyrighted material is made available to anyone wishing to use, - * modify, copy, or redistribute it subject to the terms and conditions - * of the GNU Lesser General Public License v.2.1. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/* - * This provides the interface between clvmd and gulm as the cluster - * and lock manager. - * - * It also provides the "liblm" functions too as it's hard (and pointless) - * to seperate them out when using gulm. - * - * What it does /not/ provide is the communications between clvmd daemons - * on the cluster nodes. That is done in tcp-comms.c - */ - -#include "clvmd-common.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "locking.h" -#include "clvm.h" -#include "clvmd-comms.h" -#include "lvm-functions.h" -#include "clvmd.h" -#include "clvmd-gulm.h" - -/* Hash list of nodes in the cluster */ -static struct dm_hash_table *node_hash; - -/* hash list of outstanding lock requests */ -static struct dm_hash_table *lock_hash; - -/* Copy of the current quorate state */ -static uint8_t gulm_quorate = 0; -static enum {INIT_NOTDONE, INIT_DONE, INIT_WAITQUORATE} init_state = INIT_NOTDONE; - -/* Number of active nodes */ -static int num_nodes; - -static char *cluster_name; -static int in_shutdown = 0; - -static pthread_mutex_t lock_start_mutex; -static volatile int lock_start_flag; - -struct node_info -{ - enum {NODE_UNKNOWN, NODE_DOWN, NODE_UP, NODE_CLVMD} state; - char name[GULM_MAX_CLUSTER_MEMBER_NAME_LEN]; -}; - -struct lock_wait -{ - pthread_cond_t cond; - pthread_mutex_t mutex; - int status; -}; - -/* Forward */ -static int read_from_core_sock(struct local_client *client, char *buf, int len, const char *csid, - struct local_client **new_client); -static int read_from_lock_sock(struct local_client *client, char *buf, int len, const char *csid, - struct local_client **new_client); -static int get_all_cluster_nodes(void); -static int _csid_from_name(char *csid, const char *name); -static void _cluster_closedown(void); - -/* In tcp-comms.c */ -extern struct dm_hash_table *sock_hash; - -static int add_internal_client(int fd, fd_callback_t callback) -{ - struct local_client *client; - - DEBUGLOG("Add_internal_client, fd = %d\n", fd); - - /* Add a GULM file descriptor it to the main loop */ - client = malloc(sizeof(struct local_client)); - if (!client) - { - DEBUGLOG("malloc failed\n"); - return -1; - } - - memset(client, 0, sizeof(struct local_client)); - client->fd = fd; - client->type = CLUSTER_INTERNAL; - client->callback = callback; - add_client(client); - - /* Set Close-on-exec */ - fcntl(fd, F_SETFD, 1); - - return 0; -} - -/* Gulm library handle */ -static gulm_interface_p gulm_if; -static lg_core_callbacks_t core_callbacks; -static lg_lockspace_callbacks_t lock_callbacks; - -static void badsig_handler(int sig) -{ - DEBUGLOG("got sig %d\n", sig); - _cluster_closedown(); - exit(0); -} - -static void _reread_config(void) -{ - /* Re-read CCS node list */ - DEBUGLOG("Re-reading CCS config\n"); - get_all_cluster_nodes(); -} - -static int _init_cluster(void) -{ - int status; - int ccs_h; - int port = 0; - char *portstr; - - /* Get cluster name from CCS */ - ccs_h = ccs_force_connect(NULL, 0); - if (ccs_h < 0) - { - syslog(LOG_ERR, "Cannot login in to CCSD server\n"); - return -1; - } - - ccs_get(ccs_h, "//cluster/@name", &cluster_name); - DEBUGLOG("got cluster name %s\n", cluster_name); - - if (!ccs_get(ccs_h, "//cluster/clvm/@port", &portstr)) - { - port = atoi(portstr); - free(portstr); - DEBUGLOG("got port number %d\n", port); - - if (port <= 0 && port >= 65536) - port = 0; - } - - ccs_disconnect(ccs_h); - - /* Block locking until we are logged in */ - pthread_mutex_init(&lock_start_mutex, NULL); - pthread_mutex_lock(&lock_start_mutex); - lock_start_flag = 1; - - node_hash = dm_hash_create(100); - lock_hash = dm_hash_create(10); - - /* Get all nodes from CCS */ - if (get_all_cluster_nodes()) - return -1; - - /* Initialise GULM library */ - status = lg_initialize(&gulm_if, cluster_name, "clvmd"); - if (status) - { - DEBUGLOG("lg_initialize failed: %d\n", status); - return status; - } - - /* Connect to core - we are not "important" :-) */ - status = lg_core_login(gulm_if, 0); - if (status) - { - DEBUGLOG("lg_core_login failed: %d\n", status); - return status; - } - - /* Initialise the inter-node comms */ - status = init_comms(port); - if (status) - return status; - - /* Add core FD to the list */ - status = add_internal_client(lg_core_selector(gulm_if), read_from_core_sock); - if (status) - { - DEBUGLOG("can't allocate client space\n"); - return status; - } - - /* Connect to the lock server */ - if (lg_lock_login(gulm_if, "CLVM")) - { - syslog(LOG_ERR, "Cannot login in to LOCK server\n"); - DEBUGLOG("Cannot login in to LOCK server\n"); - exit(88); - } - - /* Add lockspace FD to the list */ - status = add_internal_client(lg_lock_selector(gulm_if), read_from_lock_sock); - if (status) - { - DEBUGLOG("can't allocate client space\n"); - exit(status); - } - - /* Request a list of nodes, we can't really do anything until - this comes back */ - status = lg_core_nodelist(gulm_if); - if (status) - { - DEBUGLOG("lg_core_nodelist failed: %d\n", status); - return status; - } - - /* So I can kill it without taking GULM down too */ - signal(SIGINT, badsig_handler); - signal(SIGTERM, badsig_handler); - - return 0; -} - -static void _cluster_closedown(void) -{ - DEBUGLOG("cluster_closedown\n"); - in_shutdown = 1; - destroy_lvhash(); - lg_lock_logout(gulm_if); - lg_core_logout(gulm_if); - lg_release(gulm_if); -} - -/* Expire locks for a named node, or us */ -#define GIO_KEY_SIZE 46 -static void drop_expired_locks(char *nodename) -{ - struct utsname nodeinfo; - uint8_t mask[GIO_KEY_SIZE]; - - DEBUGLOG("Dropping expired locks for %s\n", nodename?nodename:"(null)"); - memset(mask, 0xff, GIO_KEY_SIZE); - - if (!nodename) - { - uname(&nodeinfo); - nodename = nodeinfo.nodename; - } - - if (lg_lock_drop_exp(gulm_if, nodename, mask, GIO_KEY_SIZE)) - { - DEBUGLOG("Error calling lg_lock_drop_exp()\n"); - } -} - - -static int read_from_core_sock(struct local_client *client, char *buf, int len, const char *csid, - struct local_client **new_client) -{ - int status; - - *new_client = NULL; - status = lg_core_handle_messages(gulm_if, &core_callbacks, NULL); - return status<0 ? status : 1; -} - -static int read_from_lock_sock(struct local_client *client, char *buf, int len, const char *csid, - struct local_client **new_client) -{ - int status; - - *new_client = NULL; - status = lg_lock_handle_messages(gulm_if, &lock_callbacks, NULL); - return status<0 ? status : 1; -} - - -/* CORE callback routines */ -static int core_login_reply(void *misc, uint64_t gen, uint32_t error, uint32_t rank, uint8_t corestate) -{ - DEBUGLOG("CORE Got a Login reply. gen:%lld err:%d rank:%d corestate:%d\n", - gen, error, rank, corestate); - - if (error) - exit(error); - - /* Get the current core state (for quorum) */ - lg_core_corestate(gulm_if); - - return 0; -} - -static void set_node_state(struct node_info *ninfo, char *csid, uint8_t nodestate) -{ - if (nodestate == lg_core_Logged_in) - { - /* Don't clobber NODE_CLVMD state */ - if (ninfo->state != NODE_CLVMD) - { - if (ninfo->state == NODE_UNKNOWN || - ninfo->state == NODE_DOWN) - num_nodes++; - - ninfo->state = NODE_UP; - } - } - else - { - if (nodestate == lg_core_Expired || - nodestate == lg_core_Fenced || - nodestate == lg_core_Logged_out) - { - if (ninfo->state != NODE_DOWN) - num_nodes--; - ninfo->state = NODE_DOWN; - } - } - /* Gulm doesn't always send node DOWN events, so even if this a a node UP we must - * assume (ahem) that it prevously went down at some time. So we close - * the sockets here to make sure that we don't have any dead connections - * to that node. - */ - tcp_remove_client(csid); - - DEBUGLOG("set_node_state, '%s' state = %d num_nodes=%d\n", - ninfo->name, ninfo->state, num_nodes); -} - -static struct node_info *add_or_set_node(char *name, struct in6_addr *ip, uint8_t state) -{ - struct node_info *ninfo; - - ninfo = dm_hash_lookup_binary(node_hash, (char *)ip, GULM_MAX_CSID_LEN); - if (!ninfo) - { - /* If we can't find that node then re-read the config file in case it - was added after we were started */ - DEBUGLOG("Node %s not found, re-reading config file\n", name); - get_all_cluster_nodes(); - - /* Now try again */ - ninfo = dm_hash_lookup_binary(node_hash, (char *)ip, GULM_MAX_CSID_LEN); - if (!ninfo) - { - DEBUGLOG("Ignoring node %s, not part of the SAN cluster\n", name); - return NULL; - } - } - - set_node_state(ninfo, (char *)ip, state); - - return ninfo; -} - -static void _get_our_csid(char *csid) -{ - get_our_gulm_csid(csid); -} - -static int core_nodelist(void *misc, lglcb_t type, char *name, struct in6_addr *ip, uint8_t state) -{ - DEBUGLOG("CORE nodelist\n"); - - if (type == lglcb_start) - { - DEBUGLOG("Got Nodelist, start\n"); - } - else - { - if (type == lglcb_item) - { - DEBUGLOG("Got nodelist, item: %s, %#x\n", name, state); - - add_or_set_node(name, ip, state); - } - else - { - if (type == lglcb_stop) - { - char ourcsid[GULM_MAX_CSID_LEN]; - - DEBUGLOG("Got Nodelist, stop\n"); - if (gulm_quorate) - { - clvmd_cluster_init_completed(); - init_state = INIT_DONE; - } - else - { - if (init_state == INIT_NOTDONE) - init_state = INIT_WAITQUORATE; - } - - /* Mark ourself as up */ - _get_our_csid(ourcsid); - gulm_add_up_node(ourcsid); - } - else - { - DEBUGLOG("Unknown lglcb_t %#x\n", type); - } - } - } - - return 0; -} - -static int core_statechange(void *misc, uint8_t corestate, uint8_t quorate, struct in6_addr *masterip, char *mastername) -{ - DEBUGLOG("CORE Got statechange. quorate:%d, corestate:%x mastername:%s\n", - quorate, corestate, mastername); - - gulm_quorate = quorate; - if (quorate && init_state == INIT_WAITQUORATE) - { - clvmd_cluster_init_completed(); - init_state = INIT_DONE; - } - return 0; -} - -static int core_nodechange(void *misc, char *nodename, struct in6_addr *nodeip, uint8_t nodestate) -{ - struct node_info *ninfo; - - DEBUGLOG("CORE node change, name=%s, state = %d\n", nodename, nodestate); - - /* If we don't get nodeip here, try a lookup by name */ - if (!nodeip) - _csid_from_name((char *)nodeip, nodename); - if (!nodeip) - return 0; - - ninfo = add_or_set_node(nodename, nodeip, nodestate); - if (!ninfo) - return 0; - - /* Check if we need to drop any expired locks */ - if (ninfo->state == NODE_DOWN) - { - drop_expired_locks(nodename); - } - - return 0; -} -static int core_error(void *misc, uint32_t err) -{ - DEBUGLOG("CORE error: %d\n", err); - // Not sure what happens here - return 0; -} - -/* LOCK callback routines */ -static int lock_login_reply(void *misc, uint32_t error, uint8_t which) -{ - DEBUGLOG("LOCK Got a Login reply. err:%d which:%d\n", - error, which); - - if (error) - exit(error); - - /* Drop any expired locks for us that might be hanging around */ - drop_expired_locks(NULL); - - /* Enable locking operations in other threads */ - if (lock_start_flag) - { - lock_start_flag = 0; - pthread_mutex_unlock(&lock_start_mutex); - } - - return 0; -} - -static int lock_lock_state(void *misc, uint8_t *key, uint16_t keylen, - uint64_t subid, uint64_t start, uint64_t stop, - uint8_t state, uint32_t flags, uint32_t error, - uint8_t *LVB, uint16_t LVBlen) -{ - struct lock_wait *lwait; - - DEBUGLOG("LOCK lock state: %s, error = %d\n", key, error); - - /* No waiting process to wake up when we are shutting down */ - if (in_shutdown) - return 0; - - lwait = dm_hash_lookup(lock_hash, key); - if (!lwait) - { - DEBUGLOG("Can't find hash entry for resource %s\n", key); - return 0; - } - lwait->status = error; - pthread_mutex_lock(&lwait->mutex); - pthread_cond_signal(&lwait->cond); - pthread_mutex_unlock(&lwait->mutex); - - return 0; -} -static int lock_error(void *misc, uint32_t err) -{ - DEBUGLOG("LOCK error: %d\n", err); - // Not sure what happens here - return 0; -} - - -/* CORE callbacks */ -static lg_core_callbacks_t core_callbacks = { - .login_reply = core_login_reply, - .nodelist = core_nodelist, - .statechange = core_statechange, - .nodechange = core_nodechange, - .error = core_error, -}; - -/* LOCK callbacks */ -static lg_lockspace_callbacks_t lock_callbacks = { - .login_reply = lock_login_reply, - .lock_state = lock_lock_state, - .error = lock_error, -}; - -/* Allow tcp-comms to loop round the list of active nodes */ -int get_next_node_csid(void **context, char *csid) -{ - struct node_info *ninfo = NULL; - - /* First node */ - if (!*context) - { - *context = dm_hash_get_first(node_hash); - } - else - { - *context = dm_hash_get_next(node_hash, *context); - } - if (*context) - ninfo = dm_hash_get_data(node_hash, *context); - - /* Find a node that is UP */ - while (*context && ninfo->state == NODE_DOWN) - { - *context = dm_hash_get_next(node_hash, *context); - if (*context) - { - ninfo = dm_hash_get_data(node_hash, *context); - } - } - - if (!*context || ninfo->state == NODE_DOWN) - { - return 0; - } - - memcpy(csid, dm_hash_get_key(node_hash, *context), GULM_MAX_CSID_LEN); - return 1; -} - -int gulm_name_from_csid(const char *csid, char *name) -{ - struct node_info *ninfo; - - ninfo = dm_hash_lookup_binary(node_hash, csid, GULM_MAX_CSID_LEN); - if (!ninfo) - { - sprintf(name, "UNKNOWN %s", print_csid(csid)); - return -1; - } - - strcpy(name, ninfo->name); - return 0; -} - - -static int _csid_from_name(char *csid, const char *name) -{ - struct dm_hash_node *hn; - struct node_info *ninfo; - - dm_hash_iterate(hn, node_hash) - { - ninfo = dm_hash_get_data(node_hash, hn); - if (strcmp(ninfo->name, name) == 0) - { - memcpy(csid, dm_hash_get_key(node_hash, hn), GULM_MAX_CSID_LEN); - return 0; - } - } - return -1; -} - -static int _get_num_nodes() -{ - DEBUGLOG("num_nodes = %d\n", num_nodes); - return num_nodes; -} - -/* Node is now known to be running a clvmd */ -void gulm_add_up_node(const char *csid) -{ - struct node_info *ninfo; - - ninfo = dm_hash_lookup_binary(node_hash, csid, GULM_MAX_CSID_LEN); - if (!ninfo) { - DEBUGLOG("gulm_add_up_node no node_hash entry for csid %s\n", print_csid(csid)); - return; - } - - DEBUGLOG("gulm_add_up_node %s\n", ninfo->name); - - if (ninfo->state == NODE_DOWN) - num_nodes++; - ninfo->state = NODE_CLVMD; - - return; - -} -/* Node is now known to be NOT running a clvmd */ -void add_down_node(char *csid) -{ - struct node_info *ninfo; - - ninfo = dm_hash_lookup_binary(node_hash, csid, GULM_MAX_CSID_LEN); - if (!ninfo) - return; - - /* Only set it to UP if it was previously known to be - running clvmd - gulm may set it DOWN quite soon */ - if (ninfo->state == NODE_CLVMD) - ninfo->state = NODE_UP; - drop_expired_locks(ninfo->name); - return; - -} - -/* Call a callback for each node, so the caller knows whether it's up or down */ -static int _cluster_do_node_callback(struct local_client *master_client, - void (*callback)(struct local_client *, const char *csid, int node_up)) -{ - struct dm_hash_node *hn; - struct node_info *ninfo; - int somedown = 0; - - dm_hash_iterate(hn, node_hash) - { - char csid[GULM_MAX_CSID_LEN]; - struct local_client *client; - - ninfo = dm_hash_get_data(node_hash, hn); - memcpy(csid, dm_hash_get_key(node_hash, hn), GULM_MAX_CSID_LEN); - - DEBUGLOG("down_callback. node %s, state = %d\n", ninfo->name, ninfo->state); - - client = dm_hash_lookup_binary(sock_hash, csid, GULM_MAX_CSID_LEN); - if (!client) - { - /* If it's up but not connected, try to make contact */ - if (ninfo->state == NODE_UP) - gulm_connect_csid(csid, &client); - - client = dm_hash_lookup_binary(sock_hash, csid, GULM_MAX_CSID_LEN); - - } - DEBUGLOG("down_callback2. node %s, state = %d\n", ninfo->name, ninfo->state); - if (ninfo->state != NODE_DOWN) - callback(master_client, csid, ninfo->state == NODE_CLVMD); - - if (ninfo->state != NODE_CLVMD) - somedown = -1; - } - return somedown; -} - -/* Convert gulm error codes to unix errno numbers */ -static int gulm_to_errno(int gulm_ret) -{ - switch (gulm_ret) - { - case lg_err_TryFailed: - case lg_err_AlreadyPend: - errno = EAGAIN; - break; - - /* More?? */ - default: - errno = EINVAL; - } - - return gulm_ret ? -1 : 0; -} - -/* Real locking */ -static int _lock_resource(char *resource, int mode, int flags, int *lockid) -{ - int status; - struct lock_wait lwait; - - /* Wait until the lock module is ready */ - if (lock_start_flag) - { - pthread_mutex_lock(&lock_start_mutex); - pthread_mutex_unlock(&lock_start_mutex); - } - - pthread_cond_init(&lwait.cond, NULL); - pthread_mutex_init(&lwait.mutex, NULL); - pthread_mutex_lock(&lwait.mutex); - - /* This needs to be converted from DLM/LVM2 value for GULM */ - if (flags & LCKF_NOQUEUE) flags = lg_lock_flag_Try; - - dm_hash_insert(lock_hash, resource, &lwait); - DEBUGLOG("lock_resource '%s', flags=%d, mode=%d\n", resource, flags, mode); - - status = lg_lock_state_req(gulm_if, resource, strlen(resource)+1, - 0, 0, 0, - mode, flags, NULL, 0); - if (status) - { - DEBUGLOG("lg_lock_state returned %d\n", status); - return status; - } - - /* Wait for it to complete */ - pthread_cond_wait(&lwait.cond, &lwait.mutex); - pthread_mutex_unlock(&lwait.mutex); - - dm_hash_remove(lock_hash, resource); - DEBUGLOG("lock-resource returning %d\n", lwait.status); - - return gulm_to_errno(lwait.status); -} - - -static int _unlock_resource(char *resource, int lockid) -{ - int status; - struct lock_wait lwait; - - pthread_cond_init(&lwait.cond, NULL); - pthread_mutex_init(&lwait.mutex, NULL); - pthread_mutex_lock(&lwait.mutex); - - dm_hash_insert(lock_hash, resource, &lwait); - - DEBUGLOG("unlock_resource %s\n", resource); - status = lg_lock_state_req(gulm_if, resource, strlen(resource)+1, - 0, 0, 0, - lg_lock_state_Unlock, 0, NULL, 0); - - if (status) - { - DEBUGLOG("lg_lock_state(unlock) returned %d\n", status); - return status; - } - - /* When we are shutting down, don't wait for unlocks - to be acknowledged, just do it. */ - if (in_shutdown) - return status; - - /* Wait for it to complete */ - - pthread_cond_wait(&lwait.cond, &lwait.mutex); - pthread_mutex_unlock(&lwait.mutex); - - dm_hash_remove(lock_hash, resource); - - return gulm_to_errno(lwait.status); -} - - -/* These two locking functions MUST be called in a seperate thread from - the clvmd main loop because they expect to be woken up by it. - - These are abstractions around the real locking functions (above) - as we need to emulate the DLM's EX/PW/CW interaction with GULM using - two locks. - To aid unlocking, we store the lock mode in the lockid (as GULM - doesn't use this). -*/ -static int _sync_lock(const char *resource, int mode, int flags, int *lockid) -{ - int status; - char lock1[strlen(resource)+3]; - char lock2[strlen(resource)+3]; - - snprintf(lock1, sizeof(lock1), "%s-1", resource); - snprintf(lock2, sizeof(lock2), "%s-2", resource); - - switch (mode) - { - case LCK_EXCL: - status = _lock_resource(lock1, lg_lock_state_Exclusive, flags, lockid); - if (status) - goto out; - - /* If we can't get this lock too then bail out */ - status = _lock_resource(lock2, lg_lock_state_Exclusive, LCK_NONBLOCK, lockid); - if (status == lg_err_TryFailed) - { - _unlock_resource(lock1, *lockid); - status = -1; - errno = EAGAIN; - } - break; - - case LCK_PREAD: - case LCK_READ: - status = _lock_resource(lock1, lg_lock_state_Shared, flags, lockid); - if (status) - goto out; - status = _unlock_resource(lock2, *lockid); - break; - - case LCK_WRITE: - status = _lock_resource(lock2, lg_lock_state_Exclusive, flags, lockid); - if (status) - goto out; - status = _unlock_resource(lock1, *lockid); - break; - - default: - status = -1; - errno = EINVAL; - break; - } - out: - *lockid = mode; - return status; -} - -static int _sync_unlock(const char *resource, int lockid) -{ - int status = 0; - char lock1[strlen(resource)+3]; - char lock2[strlen(resource)+3]; - - snprintf(lock1, sizeof(lock1), "%s-1", resource); - snprintf(lock2, sizeof(lock2), "%s-2", resource); - - /* The held lock mode is in the lock id */ - assert(lockid == LCK_EXCL || - lockid == LCK_READ || - lockid == LCK_PREAD || - lockid == LCK_WRITE); - - status = _unlock_resource(lock1, lockid); - if (!status) - status = _unlock_resource(lock2, lockid); - - return status; -} - -static int _is_quorate() -{ - return gulm_quorate; -} - -/* Get all the cluster node names & IPs from CCS and - add them to our node list so we know who to talk to. - Called when we start up and if we get sent SIGHUP. -*/ -static int get_all_cluster_nodes() -{ - int ctree; - char *nodename; - int error; - int i; - - /* Open the config file */ - ctree = ccs_force_connect(NULL, 1); - if (ctree < 0) - { - log_error("Error connecting to CCS"); - return -1; - } - - for (i=1;;i++) - { - char nodekey[256]; - char nodeip[GULM_MAX_CSID_LEN]; - int clvmflag = 1; - char *clvmflagstr; - char key[256]; - - sprintf(nodekey, "//cluster/clusternodes/clusternode[%d]/@name", i); - error = ccs_get(ctree, nodekey, &nodename); - if (error) - break; - - sprintf(key, "//cluster/clusternodes/clusternode[@name=\"%s\"]/clvm", nodename); - if (!ccs_get(ctree, key, &clvmflagstr)) - { - clvmflag = atoi(clvmflagstr); - free(clvmflagstr); - } - - DEBUGLOG("Got node %s from ccs(clvmflag = %d)\n", nodename, clvmflag); - if ((get_ip_address(nodename, nodeip) == 0) && clvmflag) - { - struct node_info *ninfo; - - /* If it's not in the list, then add it */ - ninfo = dm_hash_lookup_binary(node_hash, nodeip, GULM_MAX_CSID_LEN); - if (!ninfo) - { - ninfo = malloc(sizeof(struct node_info)); - if (!ninfo) - { - syslog(LOG_ERR, "Cannot alloc memory for node info\n"); - ccs_disconnect(ctree); - return -1; - } - strcpy(ninfo->name, nodename); - - ninfo->state = NODE_DOWN; - dm_hash_insert_binary(node_hash, nodeip, GULM_MAX_CSID_LEN, ninfo); - } - } - else - { - if (!clvmflag) { - DEBUGLOG("node %s has clvm disabled\n", nodename); - } - else { - DEBUGLOG("Cannot resolve host name %s\n", nodename); - log_error("Cannot resolve host name %s\n", nodename); - } - } - free(nodename); - } - - /* Finished with config file */ - ccs_disconnect(ctree); - - return 0; -} - -static int _get_main_cluster_fd(void) -{ - return get_main_gulm_cluster_fd(); -} - -static int _cluster_fd_callback(struct local_client *fd, char *buf, int len, const char *csid, struct local_client **new_client) -{ - return cluster_fd_gulm_callback(fd, buf, len, csid, new_client); -} - -static int _cluster_send_message(const void *buf, int msglen, const char *csid, const char *errtext) -{ - return gulm_cluster_send_message((char *)buf, msglen, csid, errtext); -} - -static int _get_cluster_name(char *buf, int buflen) -{ - strncpy(buf, cluster_name, buflen); - return 0; -} - -static struct cluster_ops _cluster_gulm_ops = { - .cluster_init_completed = NULL, - .cluster_send_message = _cluster_send_message, - .name_from_csid = gulm_name_from_csid, - .csid_from_name = _csid_from_name, - .get_num_nodes = _get_num_nodes, - .cluster_fd_callback = _cluster_fd_callback, - .get_main_cluster_fd = _get_main_cluster_fd, - .cluster_do_node_callback = _cluster_do_node_callback, - .is_quorate = _is_quorate, - .get_our_csid = _get_our_csid, - .add_up_node = gulm_add_up_node, - .reread_config = _reread_config, - .cluster_closedown = _cluster_closedown, - .get_cluster_name = _get_cluster_name, - .sync_lock = _sync_lock, - .sync_unlock = _sync_unlock, -}; - -struct cluster_ops *init_gulm_cluster(void) -{ - if (!_init_cluster()) - return &_cluster_gulm_ops; - else - return NULL; -} diff --git a/daemons/clvmd/clvmd-gulm.h b/daemons/clvmd/clvmd-gulm.h deleted file mode 100644 index 9416f5c39..000000000 --- a/daemons/clvmd/clvmd-gulm.h +++ /dev/null @@ -1,9 +0,0 @@ -extern int get_next_node_csid(void **context, char *csid); -extern void add_down_node(char *csid); -extern int gulm_fd(void); -extern int get_ip_address(const char *node, char *addr); -extern void tcp_remove_client(const char *csid); -extern int alloc_client(int fd, const char *csid, struct local_client **new_client); - -void gulm_add_up_node(const char *csid); -int gulm_name_from_csid(const char *csid, char *name); diff --git a/daemons/clvmd/clvmd.c b/daemons/clvmd/clvmd.c index 703268f11..07883eaa7 100644 --- a/daemons/clvmd/clvmd.c +++ b/daemons/clvmd/clvmd.c @@ -78,7 +78,6 @@ struct lvm_thread_cmd { }; struct lvm_startup_params { - int using_gulm; char **argv; }; @@ -101,7 +100,7 @@ static int child_pipe[2]; #define DFAIL_TIMEOUT 5 #define SUCCESS 0 -typedef enum {IF_AUTO, IF_CMAN, IF_GULM, IF_OPENAIS, IF_COROSYNC, IF_SINGLENODE} if_type_t; +typedef enum {IF_AUTO, IF_CMAN, IF_OPENAIS, IF_COROSYNC, IF_SINGLENODE} if_type_t; /* Prototypes for code further down */ static void sigusr2_handler(int sig); @@ -166,9 +165,6 @@ static void usage(const char *prog, FILE *file) #ifdef USE_OPENAIS "openais " #endif -#ifdef USE_GULM - "gulm " -#endif #ifdef USE_SINGLENODE "singlenode " #endif @@ -342,7 +338,6 @@ int main(int argc, char *argv[]) int start_timeout = 0; if_type_t cluster_iface = IF_AUTO; sigset_t ss; - int using_gulm = 0; int debug_opt = 0; debug_t debug_arg = DEBUG_OFF; int clusterwide_opt = 0; @@ -473,7 +468,7 @@ int main(int argc, char *argv[]) /* Set up signal handlers, USR1 is for cluster change notifications (in cman) USR2 causes child threads to exit. - HUP causes gulm version to re-read nodes list from CCS. + HUP causes to re-read nodes list from CCS. PIPE should be ignored */ signal(SIGUSR2, sigusr2_handler); signal(SIGHUP, sighup_handler); @@ -505,16 +500,6 @@ int main(int argc, char *argv[]) syslog(LOG_NOTICE, "Cluster LVM daemon started - connected to CMAN"); } #endif -#ifdef USE_GULM - if (!clops) - if ((cluster_iface == IF_AUTO || cluster_iface == IF_GULM) && (clops = init_gulm_cluster())) { - max_csid_len = GULM_MAX_CSID_LEN; - max_cluster_message = GULM_MAX_CLUSTER_MESSAGE; - max_cluster_member_name_len = GULM_MAX_CLUSTER_MEMBER_NAME_LEN; - using_gulm = 1; - syslog(LOG_NOTICE, "Cluster LVM daemon started - connected to GULM"); - } -#endif #ifdef USE_COROSYNC if (!clops) if (((cluster_iface == IF_AUTO || cluster_iface == IF_COROSYNC) && (clops = init_corosync_cluster()))) { @@ -580,14 +565,10 @@ int main(int argc, char *argv[]) /* Don't let anyone else to do work until we are started */ pthread_mutex_lock(&lvm_start_mutex); - lvm_params.using_gulm = using_gulm; lvm_params.argv = argv; pthread_create(&lvm_thread, NULL, lvm_thread_fn, &lvm_params); /* Tell the rest of the cluster our version number */ - /* CMAN can do this immediately, gulm needs to wait until - the core initialisation has finished and the node list - has been gathered */ if (clops->cluster_init_completed) clops->cluster_init_completed(); @@ -625,7 +606,7 @@ int main(int argc, char *argv[]) return 0; } -/* Called when the GuLM cluster layer has completed initialisation. +/* Called when the cluster layer has completed initialisation. We send the version message */ void clvmd_cluster_init_completed(void) { @@ -1965,7 +1946,7 @@ static void *lvm_thread_fn(void *arg) pthread_sigmask(SIG_BLOCK, &ss, NULL); /* Initialise the interface to liblvm */ - init_clvm(lvm_params->using_gulm, lvm_params->argv); + init_clvm(lvm_params->argv); /* Allow others to get moving */ pthread_mutex_unlock(&lvm_start_mutex); @@ -2222,8 +2203,6 @@ static if_type_t parse_cluster_interface(char *ifname) iface = IF_AUTO; if (!strcmp(ifname, "cman")) iface = IF_CMAN; - if (!strcmp(ifname, "gulm")) - iface = IF_GULM; if (!strcmp(ifname, "openais")) iface = IF_OPENAIS; if (!strcmp(ifname, "corosync")) diff --git a/daemons/clvmd/lvm-functions.c b/daemons/clvmd/lvm-functions.c index c10ce9df3..d2c89d9ad 100644 --- a/daemons/clvmd/lvm-functions.c +++ b/daemons/clvmd/lvm-functions.c @@ -650,46 +650,6 @@ int do_refresh_cache(void) return 0; } - -/* Only called at gulm startup. Drop any leftover VG or P_orphan locks - that might be hanging around if we died for any reason -*/ -static void drop_vg_locks(void) -{ - char vg[128]; - char line[255]; - FILE *vgs = - popen - (LVM_PATH " pvs --config 'log{command_names=0 prefix=\"\"}' --nolocking --noheadings -o vg_name", "r"); - - sync_unlock("P_" VG_ORPHANS, LCK_EXCL); - sync_unlock("P_" VG_GLOBAL, LCK_EXCL); - - if (!vgs) - return; - - while (fgets(line, sizeof(line), vgs)) { - char *vgend; - char *vgstart; - - if (line[strlen(line)-1] == '\n') - line[strlen(line)-1] = '\0'; - - vgstart = line + strspn(line, " "); - vgend = vgstart + strcspn(vgstart, " "); - *vgend = '\0'; - - if (strncmp(vgstart, "WARNING:", 8) == 0) - continue; - - sprintf(vg, "V_%s", vgstart); - sync_unlock(vg, LCK_EXCL); - - } - if (fclose(vgs)) - DEBUGLOG("vgs fclose failed: %s\n", strerror(errno)); -} - /* * Handle VG lock - drop metadata or update lvmcache state */ @@ -920,7 +880,7 @@ void lvm_do_fs_unlock(void) } /* Called to initialise the LVM context of the daemon */ -int init_clvm(int using_gulm, char **argv) +int init_clvm(char **argv) { /* Use LOG_DAEMON for syslog messages instead of LOG_USER */ init_syslog(LOG_DAEMON); @@ -942,10 +902,6 @@ int init_clvm(int using_gulm, char **argv) check_config(); init_ignore_suspended_devices(1); - /* Remove any non-LV locks that may have been left around */ - if (using_gulm) - drop_vg_locks(); - get_initial_state(argv); /* Trap log messages so we can pass them back to the user */ diff --git a/daemons/clvmd/lvm-functions.h b/daemons/clvmd/lvm-functions.h index a132f4af7..f9c43b7ce 100644 --- a/daemons/clvmd/lvm-functions.h +++ b/daemons/clvmd/lvm-functions.h @@ -27,7 +27,7 @@ extern int post_lock_lv(unsigned char lock_cmd, unsigned char lock_flags, char *resource); extern int do_check_lvm1(const char *vgname); extern int do_refresh_cache(void); -extern int init_clvm(int using_gulm, char **argv); +extern int init_clvm(char **argv); extern void destroy_lvm(void); extern void init_lvhash(void); extern void destroy_lvhash(void); diff --git a/daemons/clvmd/tcp-comms.c b/daemons/clvmd/tcp-comms.c deleted file mode 100644 index 5f8655665..000000000 --- a/daemons/clvmd/tcp-comms.c +++ /dev/null @@ -1,502 +0,0 @@ -/* - * Copyright (C) 2002-2003 Sistina Software, Inc. All rights reserved. - * Copyright (C) 2004-2009 Red Hat, Inc. All rights reserved. - * - * This file is part of LVM2. - * - * This copyrighted material is made available to anyone wishing to use, - * modify, copy, or redistribute it subject to the terms and conditions - * of the GNU Lesser General Public License v.2.1. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/* - * This provides the inter-clvmd communications for a system without CMAN. - * There is a listening TCP socket which accepts new connections in the - * normal way. - * It can also make outgoing connnections to the other clvmd nodes. - */ - -#include "clvmd-common.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "clvm.h" -#include "clvmd-comms.h" -#include "clvmd.h" -#include "clvmd-gulm.h" - -#define DEFAULT_TCP_PORT 21064 - -static int listen_fd = -1; -static int tcp_port; -struct dm_hash_table *sock_hash; - -static int get_our_ip_address(char *addr, int *family); -static int read_from_tcpsock(struct local_client *fd, char *buf, int len, char *csid, - struct local_client **new_client); - -/* Called by init_cluster() to open up the listening socket */ -int init_comms(unsigned short port) -{ - struct sockaddr_in6 addr; - - sock_hash = dm_hash_create(100); - tcp_port = port ? : DEFAULT_TCP_PORT; - - listen_fd = socket(AF_INET6, SOCK_STREAM, 0); - - if (listen_fd < 0) - { - return -1; - } - else - { - int one = 1; - setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(int)); - setsockopt(listen_fd, SOL_SOCKET, SO_KEEPALIVE, &one, sizeof(int)); - } - - memset(&addr, 0, sizeof(addr)); // Bind to INADDR_ANY - addr.sin6_family = AF_INET6; - addr.sin6_port = htons(tcp_port); - - if (bind(listen_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) - { - DEBUGLOG("Can't bind to port: %s\n", strerror(errno)); - syslog(LOG_ERR, "Can't bind to port %d, is clvmd already running ?", tcp_port); - close(listen_fd); - return -1; - } - - listen(listen_fd, 5); - - /* Set Close-on-exec */ - fcntl(listen_fd, F_SETFD, 1); - - return 0; -} - -void tcp_remove_client(const char *c_csid) -{ - struct local_client *client; - char csid[GULM_MAX_CSID_LEN]; - unsigned int i; - memcpy(csid, c_csid, sizeof csid); - DEBUGLOG("tcp_remove_client\n"); - - /* Don't actually close the socket here - that's the - job of clvmd.c whch will do the job when it notices the - other end has gone. We just need to remove the client(s) from - the hash table so we don't try to use it for sending any more */ - for (i = 0; i < 2; i++) - { - client = dm_hash_lookup_binary(sock_hash, csid, GULM_MAX_CSID_LEN); - if (client) - { - dm_hash_remove_binary(sock_hash, csid, GULM_MAX_CSID_LEN); - client->removeme = 1; - close(client->fd); - } - /* Look for a mangled one too, on the 2nd iteration. */ - csid[0] ^= 0x80; - } -} - -int alloc_client(int fd, const char *c_csid, struct local_client **new_client) -{ - struct local_client *client; - char csid[GULM_MAX_CSID_LEN]; - memcpy(csid, c_csid, sizeof csid); - - DEBUGLOG("alloc_client %d csid = %s\n", fd, print_csid(csid)); - - /* Create a local_client and return it */ - client = malloc(sizeof(struct local_client)); - if (!client) - { - DEBUGLOG("malloc failed\n"); - return -1; - } - - memset(client, 0, sizeof(struct local_client)); - client->fd = fd; - client->type = CLUSTER_DATA_SOCK; - client->callback = read_from_tcpsock; - if (new_client) - *new_client = client; - - /* Add to our list of node sockets */ - if (dm_hash_lookup_binary(sock_hash, csid, GULM_MAX_CSID_LEN)) - { - DEBUGLOG("alloc_client mangling CSID for second connection\n"); - /* This is a duplicate connection but we can't close it because - the other end may already have started sending. - So, we mangle the IP address and keep it, all sending will - go out of the main FD - */ - csid[0] ^= 0x80; - client->bits.net.flags = 1; /* indicate mangled CSID */ - - /* If it still exists then kill the connection as we should only - ever have one incoming connection from each node */ - if (dm_hash_lookup_binary(sock_hash, csid, GULM_MAX_CSID_LEN)) - { - DEBUGLOG("Multiple incoming connections from node\n"); - syslog(LOG_ERR, " Bogus incoming connection from %d.%d.%d.%d\n", csid[0],csid[1],csid[2],csid[3]); - - free(client); - errno = ECONNREFUSED; - return -1; - } - } - dm_hash_insert_binary(sock_hash, csid, GULM_MAX_CSID_LEN, client); - - return 0; -} - -int get_main_gulm_cluster_fd() -{ - return listen_fd; -} - - -/* Read on main comms (listen) socket, accept it */ -int cluster_fd_gulm_callback(struct local_client *fd, char *buf, int len, const char *csid, - struct local_client **new_client) -{ - int newfd; - struct sockaddr_in6 addr; - socklen_t addrlen = sizeof(addr); - int status; - char name[GULM_MAX_CLUSTER_MEMBER_NAME_LEN]; - - DEBUGLOG("cluster_fd_callback\n"); - *new_client = NULL; - newfd = accept(listen_fd, (struct sockaddr *)&addr, &addrlen); - - DEBUGLOG("cluster_fd_callback, newfd=%d (errno=%d)\n", newfd, errno); - if (!newfd) - { - syslog(LOG_ERR, "error in accept: %m"); - errno = EAGAIN; - return -1; /* Don't return an error or clvmd will close the listening FD */ - } - - /* Check that the client is a member of the cluster - and reject if not. - */ - if (gulm_name_from_csid((char *)&addr.sin6_addr, name) < 0) - { - syslog(LOG_ERR, "Got connect from non-cluster node %s\n", - print_csid((char *)&addr.sin6_addr)); - DEBUGLOG("Got connect from non-cluster node %s\n", - print_csid((char *)&addr.sin6_addr)); - close(newfd); - - errno = EAGAIN; - return -1; - } - - status = alloc_client(newfd, (char *)&addr.sin6_addr, new_client); - if (status) - { - DEBUGLOG("cluster_fd_callback, alloc_client failed, status = %d\n", status); - close(newfd); - /* See above... */ - errno = EAGAIN; - return -1; - } - DEBUGLOG("cluster_fd_callback, returning %d, %p\n", newfd, *new_client); - return newfd; -} - -/* Try to get at least 'len' bytes from the socket */ -static int really_read(int fd, char *buf, int len) -{ - int got, offset; - - got = offset = 0; - - do { - got = read(fd, buf+offset, len-offset); - DEBUGLOG("really_read. got %d bytes\n", got); - offset += got; - } while (got > 0 && offset < len); - - if (got < 0) - return got; - else - return offset; -} - - -static int read_from_tcpsock(struct local_client *client, char *buf, int len, char *csid, - struct local_client **new_client) -{ - struct sockaddr_in6 addr; - socklen_t slen = sizeof(addr); - struct clvm_header *header = (struct clvm_header *)buf; - int status; - uint32_t arglen; - - DEBUGLOG("read_from_tcpsock fd %d\n", client->fd); - *new_client = NULL; - - /* Get "csid" */ - getpeername(client->fd, (struct sockaddr *)&addr, &slen); - memcpy(csid, &addr.sin6_addr, GULM_MAX_CSID_LEN); - - /* Read just the header first, then get the rest if there is any. - * Stream sockets, sigh. - */ - status = really_read(client->fd, buf, sizeof(struct clvm_header)); - if (status > 0) - { - int status2; - - arglen = ntohl(header->arglen); - - /* Get the rest */ - if (arglen && arglen < GULM_MAX_CLUSTER_MESSAGE) - { - status2 = really_read(client->fd, buf+status, arglen); - if (status2 > 0) - status += status2; - else - status = status2; - } - } - - DEBUGLOG("read_from_tcpsock, status = %d(errno = %d)\n", status, errno); - - /* Remove it from the hash table if there's an error, clvmd will - remove the socket from its lists and free the client struct */ - if (status == 0 || - (status < 0 && errno != EAGAIN && errno != EINTR)) - { - char remcsid[GULM_MAX_CSID_LEN]; - - memcpy(remcsid, csid, GULM_MAX_CSID_LEN); - close(client->fd); - - /* If the csid was mangled, then make sure we remove the right entry */ - if (client->bits.net.flags) - remcsid[0] ^= 0x80; - dm_hash_remove_binary(sock_hash, remcsid, GULM_MAX_CSID_LEN); - - /* Tell cluster manager layer */ - add_down_node(remcsid); - } - else { - gulm_add_up_node(csid); - /* Send it back to clvmd */ - process_message(client, buf, status, csid); - } - return status; -} - -int gulm_connect_csid(const char *csid, struct local_client **newclient) -{ - int fd; - struct sockaddr_in6 addr; - int status; - int one = 1; - - DEBUGLOG("Connecting socket\n"); - fd = socket(PF_INET6, SOCK_STREAM, 0); - - if (fd < 0) - { - syslog(LOG_ERR, "Unable to create new socket: %m"); - return -1; - } - - addr.sin6_family = AF_INET6; - memcpy(&addr.sin6_addr, csid, GULM_MAX_CSID_LEN); - addr.sin6_port = htons(tcp_port); - - DEBUGLOG("Connecting socket %d\n", fd); - if (connect(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in6)) < 0) - { - /* "Connection refused" is "normal" because clvmd may not yet be running - * on that node. - */ - if (errno != ECONNREFUSED) - { - syslog(LOG_ERR, "Unable to connect to remote node: %m"); - } - DEBUGLOG("Unable to connect to remote node: %s\n", strerror(errno)); - close(fd); - return -1; - } - - /* Set Close-on-exec */ - fcntl(fd, F_SETFD, 1); - setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &one, sizeof(int)); - - status = alloc_client(fd, csid, newclient); - if (status) - close(fd); - else - add_client(*newclient); - - /* If we can connect to it, it must be running a clvmd */ - gulm_add_up_node(csid); - return status; -} - -/* Send a message to a known CSID */ -static int tcp_send_message(void *buf, int msglen, const char *csid, const char *errtext) -{ - int status; - struct local_client *client; - char ourcsid[GULM_MAX_CSID_LEN]; - - assert(csid); - - DEBUGLOG("tcp_send_message, csid = %s, msglen = %d\n", print_csid(csid), msglen); - - /* Don't connect to ourself */ - get_our_gulm_csid(ourcsid); - if (memcmp(csid, ourcsid, GULM_MAX_CSID_LEN) == 0) - return msglen; - - client = dm_hash_lookup_binary(sock_hash, csid, GULM_MAX_CSID_LEN); - if (!client) - { - status = gulm_connect_csid(csid, &client); - if (status) - return -1; - } - DEBUGLOG("tcp_send_message, fd = %d\n", client->fd); - - return write(client->fd, buf, msglen); -} - - -int gulm_cluster_send_message(void *buf, int msglen, const char *csid, const char *errtext) -{ - int status=0; - - DEBUGLOG("cluster send message, csid = %p, msglen = %d\n", csid, msglen); - - /* If csid is NULL then send to all known (not just connected) nodes */ - if (!csid) - { - void *context = NULL; - char loop_csid[GULM_MAX_CSID_LEN]; - - /* Loop round all gulm-known nodes */ - while (get_next_node_csid(&context, loop_csid)) - { - status = tcp_send_message(buf, msglen, loop_csid, errtext); - if (status == 0 || - (status < 0 && (errno == EAGAIN || errno == EINTR))) - break; - } - } - else - { - - status = tcp_send_message(buf, msglen, csid, errtext); - } - return status; -} - -/* To get our own IP address we get the locally bound address of the - socket that's talking to GULM in the assumption(eek) that it will - be on the "right" network in a multi-homed system */ -static int get_our_ip_address(char *addr, int *family) -{ - struct utsname info; - - uname(&info); - get_ip_address(info.nodename, addr); - - return 0; -} - -/* Public version of above for those that don't care what protocol - we're using */ -void get_our_gulm_csid(char *csid) -{ - static char our_csid[GULM_MAX_CSID_LEN]; - static int got_csid = 0; - - if (!got_csid) - { - int family; - - memset(our_csid, 0, sizeof(our_csid)); - if (get_our_ip_address(our_csid, &family)) - { - got_csid = 1; - } - } - memcpy(csid, our_csid, GULM_MAX_CSID_LEN); -} - -static void map_v4_to_v6(struct in_addr *ip4, struct in6_addr *ip6) -{ - ip6->s6_addr32[0] = 0; - ip6->s6_addr32[1] = 0; - ip6->s6_addr32[2] = htonl(0xffff); - ip6->s6_addr32[3] = ip4->s_addr; -} - -/* Get someone else's IP address from DNS */ -int get_ip_address(const char *node, char *addr) -{ - struct hostent *he; - - memset(addr, 0, GULM_MAX_CSID_LEN); - - // TODO: what do we do about multi-homed hosts ??? - // CCSs ip_interfaces solved this but some bugger removed it. - - /* Try IPv6 first. The man page for gethostbyname implies that - it will lookup ip6 & ip4 names, but it seems not to */ - he = gethostbyname2(node, AF_INET6); - if (he) - { - memcpy(addr, he->h_addr_list[0], - he->h_length); - } - else - { - he = gethostbyname2(node, AF_INET); - if (!he) - return -1; - map_v4_to_v6((struct in_addr *)he->h_addr_list[0], (struct in6_addr *)addr); - } - - return 0; -} - -char *print_csid(const char *csid) -{ - static char buf[128]; - int *icsid = (int *)csid; - - sprintf(buf, "[%x.%x.%x.%x]", - icsid[0],icsid[1],icsid[2],icsid[3]); - - return buf; -} diff --git a/daemons/clvmd/tcp-comms.h b/daemons/clvmd/tcp-comms.h deleted file mode 100644 index 9260e1235..000000000 --- a/daemons/clvmd/tcp-comms.h +++ /dev/null @@ -1,13 +0,0 @@ -#include - -#define GULM_MAX_CLUSTER_MESSAGE 1600 -#define GULM_MAX_CSID_LEN sizeof(struct in6_addr) -#define GULM_MAX_CLUSTER_MEMBER_NAME_LEN 128 - -extern int init_comms(unsigned short); -extern char *print_csid(const char *); -int get_main_gulm_cluster_fd(void); -int cluster_fd_gulm_callback(struct local_client *fd, char *buf, int len, const char *csid, struct local_client **new_client); -int gulm_cluster_send_message(void *buf, int msglen, const char *csid, const char *errtext); -void get_our_gulm_csid(char *csid); -int gulm_connect_csid(const char *csid, struct local_client **newclient); diff --git a/man/clvmd.8.in b/man/clvmd.8.in index 68dc82fdd..a4b9cc9c3 100644 --- a/man/clvmd.8.in +++ b/man/clvmd.8.in @@ -85,7 +85,7 @@ exclusive LV locks into shared locks.) Selects the cluster manager to use for locking and internal communications, the available managers will be listed as part of the 'clvmd -h' output. clvmd will use the first cluster manager that succeeds, and it checks them -in the order cman,gulm,corosync,openais. As it is quite possible to have +in the order cman,corosync,openais. As it is quite possible to have (eg) corosync and cman available on the same system you might have to manually specify this option to override the search. .TP