Can now build a gulm version of clvmd instead of a cman one. use

./configure --with-clvmd=gulm

The default is still cman, and you can't have both - sorry.

WHATS_NEW

@@ -6,6 +6,7 @@ Version 2.00.26 -
   Make clvmd cope with large gaps in node numbers IDs.
   Make clvmd initialisation cope better with debugging output.
   clvmd -V now displays lvm version too.
+  Add optional gulm build for clvmd
 
 Version 2.00.25 - 29th September 2004
 =====================================

configure.in

@@ -280,12 +280,15 @@ AC_MSG_RESULT($SELINUX)
 ################################################################################
 dnl -- Build cluster LVM daemon
 AC_MSG_CHECKING(whether to build cluster LVM daemon)
-AC_ARG_WITH(clvmd, [  --with-clvmd            Build cluster LVM Daemon],
+AC_ARG_WITH(clvmd, [  --with-clvmd=TYPE       Build cluster LVM Daemon: cman/gulm/none],
-CLVMD=$withval, CLVMD=no)
+CLVMD=$withval, CLVMD=none)
+if test x$CLVMD = xyes ; then
+	CLVMD=cman
+fi
 AC_MSG_RESULT($CLVMD)
 
 dnl -- If clvmd enabled without cluster locking, automagically include it
-if  test x$CLVMD = xyes && test x$CLUSTER = xnone; then
+if  test x$CLVMD != xnone && test x$CLUSTER = xnone ; then
 	CLUSTER=internal
 fi
 

daemons/Makefile.in

@@ -15,7 +15,7 @@ srcdir = @srcdir@
 top_srcdir = @top_srcdir@
 VPATH = @srcdir@
 
-ifeq ("@CLVMD@", "yes")
+ifneq ("@CLVMD@", "none")
   SUBDIRS = clvmd
 endif
 

daemons/clvmd/Makefile.in

@@ -15,6 +15,18 @@ srcdir = @srcdir@
 top_srcdir = @top_srcdir@
 VPATH = @srcdir@
 
+ifeq ("@CLVMD@", "gulm")
+SOURCES = \
+	clvmd-gulm.c     \
+	tcp-comms.c	 \
+        clvmd-command.c  \
+        clvmd.c          \
+        libclvm.c        \
+        lvm-functions.c  \
+        system-lv.c
+LM_LIBS = -lccs -lgulm
+LM_FLAGS= -DUSE_GULM
+else
 SOURCES = \
 	clvmd-cman.c     \
 	clvmd-command.c  \
@@ -22,20 +34,24 @@ SOURCES = \
 	libclvm.c        \
 	lvm-functions.c  \
 	system-lv.c
+LM_LIBS = -ldlm
+LM_FLAGS=
+endif
 
 TARGETS = \
 	clvmd
 
 include $(top_srcdir)/make.tmpl
 
-CFLAGS += -D_REENTRANT -fno-strict-aliasing
+CFLAGS += -D_REENTRANT -fno-strict-aliasing $(LM_FLAGS)
-LIBS += -ldevmapper -ldlm -llvm -lpthread
+LIBS += -ldevmapper -llvm -lpthread
+
 
 INSTALL_TARGETS = \
 	install_clvmd
 
 clvmd: $(OBJECTS) $(top_srcdir)/lib/liblvm.a
-	$(CC) -o clvmd $(OBJECTS) $(LDFLAGS) $(LVMLIBS) $(LIBS)
+	$(CC) -o clvmd $(OBJECTS) $(LDFLAGS) $(LVMLIBS) $(LM_LIBS) $(LIBS)
 
 .PHONY: install_clvmd
 

daemons/clvmd/clvmd-cman.c

@@ -55,7 +55,6 @@ static int max_updown_nodes = 50;	/* Current size of the allocated array */
 static int *node_updown = NULL;
 static dlm_lshandle_t *lockspace;
 
-static void sigusr1_handler(int sig);
 static void count_clvmds_running(void);
 static void get_members(void);
 static int nodeid_from_csid(char *csid);

daemons/clvmd/clvmd-gulm.c

@@ -139,11 +139,28 @@ int init_cluster()
 {
     int status;
     int ccs_h;
+    int port = 0;
+    char *portstr;
 
     /* Get cluster name from CCS */
     /* TODO: is this right? */
-    ccs_h = ccs_connect();
+    ccs_h = ccs_force_connect(NULL, 1); // PJC
+    if (!ccs_h)
+	return -1;
+
     ccs_get(ccs_h, "//cluster/@name", &cluster_name);
+    DEBUGLOG("got cluster name %s\n", cluster_name);
+
+    if (!ccs_get(ccs_h, "//clvm/@port", &portstr))
+    {
+	port = atoi(portstr);
+	free(portstr);
+	DEBUGLOG("got port number %d\n", port);
+
+	if (port <= 0 && port >= 65536)
+	    port = 0;
+    }
+
     ccs_disconnect(ccs_h);
 
     /* Block locking until we are logged in */
@@ -155,7 +172,8 @@ int init_cluster()
     lock_hash = hash_create(10);
 
     /* Get all nodes from CCS */
-    get_all_cluster_nodes();
+    if (get_all_cluster_nodes())
+	return -1;
 
     /* Initialise GULM library */
     status = lg_initialize(&gulm_if, cluster_name, "clvmd");
@@ -174,7 +192,7 @@ int init_cluster()
     }
 
     /* Initialise the inter-node comms */
-    status = init_comms();
+    status = init_comms(port);
     if (status)
 	return status;
 
@@ -316,11 +334,11 @@ static void set_node_state(struct node_info *ninfo, char *csid, uint8_t nodestat
 	     ninfo->name, ninfo->state, num_nodes);
 }
 
-static struct node_info *add_or_set_node(char *name, uint32_t ip, uint8_t state)
+static struct node_info *add_or_set_node(char *name, struct in6_addr *ip, uint8_t state)
 {
     struct node_info *ninfo;
 
-    ninfo = hash_lookup_binary(node_hash, (char *)&ip, MAX_CSID_LEN);
+    ninfo = hash_lookup_binary(node_hash, (char *)ip, MAX_CSID_LEN);
     if (!ninfo)
     {
 	/* If we can't find that node then re-read the config file in case it
@@ -329,7 +347,7 @@ static struct node_info *add_or_set_node(char *name, uint32_t ip, uint8_t state)
 	get_all_cluster_nodes();
 
 	/* Now try again */
-	ninfo = hash_lookup_binary(node_hash, (char *)&ip, MAX_CSID_LEN);
+	ninfo = hash_lookup_binary(node_hash, (char *)ip, MAX_CSID_LEN);
 	if (!ninfo)
 	{
 	    DEBUGLOG("Ignoring node %s, not part of the SAN cluster\n", name);
@@ -342,7 +360,7 @@ static struct node_info *add_or_set_node(char *name, uint32_t ip, uint8_t state)
     return ninfo;
 }
 
-static int core_nodelist(void *misc, lglcb_t type, char *name, uint32_t ip, uint8_t state)
+static int core_nodelist(void *misc, lglcb_t type, char *name, struct in6_addr *ip, uint8_t state)
 {
     DEBUGLOG("CORE nodelist\n");
 
@@ -354,7 +372,7 @@ static int core_nodelist(void *misc, lglcb_t type, char *name, uint32_t ip, uint
     {
 	if (type == lglcb_item)
 	{
-	    DEBUGLOG("Got nodelist, item: %s, %#x, %#x\n", name, ip, state);
+	    DEBUGLOG("Got nodelist, item: %s, %#x\n", name, state);
 
 	    add_or_set_node(name, ip, state);
 	}
@@ -381,24 +399,24 @@ static int core_nodelist(void *misc, lglcb_t type, char *name, uint32_t ip, uint
     return 0;
 }
 
-static int core_statechange(void *misc, uint8_t corestate, uint32_t masterip, char *mastername)
+static int core_statechange(void *misc, uint8_t corestate, uint8_t quorate, struct in6_addr *masterip, char *mastername)
 {
-    DEBUGLOG("CORE Got statechange  corestate:%#x masterip:%#x mastername:%s\n",
+    DEBUGLOG("CORE Got statechange  corestate:%#x mastername:%s\n",
-	     corestate, masterip, mastername);
+	     corestate, mastername);
 
     current_corestate = corestate;
     return 0;
 }
 
-static int core_nodechange(void *misc, char *nodename, uint32_t nodeip, uint8_t nodestate)
+static int core_nodechange(void *misc, char *nodename, struct in6_addr *nodeip, uint8_t nodestate)
 {
     struct node_info *ninfo;
 
-    DEBUGLOG("CORE node change, name=%s, ip=%x, state = %d\n", nodename, nodeip, nodestate);
+    DEBUGLOG("CORE node change, name=%s, state = %d\n", nodename, nodestate);
 
     /* If we don't get nodeip here, try a lookup by name */
     if (!nodeip)
-	csid_from_name((char *)&nodeip, nodename);
+	csid_from_name((char *)nodeip, nodename);
     if (!nodeip)
 	return 0;
 
@@ -443,7 +461,9 @@ static int lock_login_reply(void *misc, uint32_t error, uint8_t which)
     return 0;
 }
 
-static int lock_lock_state(void *misc, uint8_t *key, uint16_t keylen, uint8_t state, uint32_t flags, uint32_t error,
+static int lock_lock_state(void *misc, uint8_t *key, uint16_t keylen,
+			   uint64_t subid, uint64_t start, uint64_t stop,
+			   uint8_t state, uint32_t flags, uint32_t error,
 			   uint8_t *LVB, uint16_t LVBlen)
 {
     struct lock_wait *lwait;
@@ -530,8 +550,7 @@ int name_from_csid(char *csid, char *name)
     ninfo = hash_lookup_binary(node_hash, csid, MAX_CSID_LEN);
     if (!ninfo)
     {
-	sprintf(name, "UNKNOWN [%d.%d.%d.%d]",
+        sprintf(name, "UNKNOWN %s", print_csid(csid));
-		csid[0], csid[1], csid[2], csid[3]);
 	return -1;
     }
 
@@ -661,6 +680,7 @@ static int _lock_resource(char *resource, int mode, int flags, int *lockid)
     DEBUGLOG("lock_resource '%s', flags=%d, mode=%d\n", resource, flags, mode);
 
     status = lg_lock_state_req(gulm_if, resource, strlen(resource)+1,
+			       0, 0, 0,
 			       mode, flags, NULL, 0);
     if (status)
     {
@@ -692,6 +712,7 @@ static int _unlock_resource(char *resource, int lockid)
 
     DEBUGLOG("unlock_resource %s\n", resource);
     status = lg_lock_state_req(gulm_if, resource, strlen(resource)+1,
+			       0, 0, 0,
 			       lg_lock_state_Unlock, 0, NULL, 0);
 
     if (status)
@@ -819,26 +840,38 @@ static int get_all_cluster_nodes()
     int ctree;
     char *nodename;
     int error;
+    int i;
 
     /* Open the config file */
-    ctree = ccs_connect();
+    ctree = ccs_force_connect(NULL, 1);
     if (ctree <= 0)
     {
 	log_error("Error connecting to CCS");
 	return -1;
     }
 
-    error = ccs_get(ctree, "//nodes/node/@name", &nodename);
+    for (i=1; i++;)
-    while (nodename)
     {
+	char nodekey[256];
 	char nodeip[MAX_CSID_LEN];
-	char *clvmflag;
+	int  clvmflag = 1;
+	char *clvmflagstr;
 	char key[256];
 
-	sprintf(key, "//nodes/node[@name=\"%s\"]/clvm", nodename);
+	sprintf(nodekey, "//cluster/nodes/node[%d]/@name", i);
-	ccs_get(ctree, key, &clvmflag);
+	error = ccs_get(ctree, nodekey, &nodename);
+	if (error)
+	    break;
 
-	if ((get_ip_address(nodename, nodeip) == 0) && atoi(clvmflag))
+	sprintf(key, "//nodes/node[@name=\"%s\"]/clvm", nodename);
+	if (!ccs_get(ctree, key, &clvmflagstr))
+	{
+	    clvmflag = atoi(clvmflagstr);
+	    free(clvmflagstr);
+	}
+
+	DEBUGLOG("Got node %s from ccs(clvmflag = %d)\n", nodename, clvmflag);
+	if ((get_ip_address(nodename, nodeip) == 0) && clvmflag)
 	{
 	    struct node_info *ninfo;
 
@@ -863,7 +896,6 @@ static int get_all_cluster_nodes()
 	{
 	    DEBUGLOG("node %s has clvm disabled\n", nodename);
 	}
-	if (clvmflag) free(clvmflag);
 	free(nodename);
 	error = ccs_get(ctree, "//nodes/node/@name", &nodename);
     }

daemons/clvmd/tcp-comms.c

@@ -34,7 +34,6 @@
 #include <netdb.h>
 #include <assert.h>
 
-#include "ccs.h"
 #include "clvm.h"
 #include "clvmd-comms.h"
 #include "clvmd.h"
@@ -47,43 +46,19 @@ static int listen_fd = -1;
 static int tcp_port;
 struct hash_table *sock_hash;
 
-static int get_tcp_port(int default_port);
 static int get_our_ip_address(char *addr, int *family);
 static int read_from_tcpsock(struct local_client *fd, char *buf, int len, char *csid,
 			     struct local_client **new_client);
 
 /* Called by init_cluster() to open up the listening socket */
-// TODO: IPv6 compat.
+int init_comms(unsigned short port)
-int init_comms()
 {
-    struct sockaddr *addr = NULL;
+    struct sockaddr_in6 addr;
-    struct sockaddr_in addr4;
-    struct sockaddr_in6 addr6;
-    int    addr_len;
-    int    family;
-    char   address[MAX_CSID_LEN];
 
     sock_hash = hash_create(100);
-    tcp_port = get_tcp_port(DEFAULT_TCP_PORT);
+    tcp_port = port ? port : DEFAULT_TCP_PORT;
 
-    /* Get IP address and IP type */
+    listen_fd = socket(AF_INET6, SOCK_STREAM, 0);
-    get_our_ip_address(address, &family);
-    if (family == AF_INET)
-    {
-	memcpy(&addr4.sin_addr, addr, sizeof(struct in_addr));
-	addr = (struct sockaddr *)&addr4;
-	addr4.sin_port = htons(tcp_port);
-	addr_len = sizeof(addr4);
-    }
-    else
-    {
-	memcpy(&addr6.sin6_addr, addr, sizeof(struct in6_addr));
-	addr = (struct sockaddr *)&addr6;
-	addr6.sin6_port = htons(tcp_port);
-	addr_len = sizeof(addr6);
-    }
-
-    listen_fd = socket(family, SOCK_STREAM, 0);
 
     if (listen_fd < 0)
     {
@@ -95,11 +70,13 @@ int init_comms()
 	setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(int));
     }
 
-    addr->sa_family = family;
+    memset(&addr, 0, sizeof(addr)); // Bind to INADDR_ANY
+    addr.sin6_family = AF_INET6;
+    addr.sin6_port = htons(tcp_port);
 
-    if (bind(listen_fd, addr, addr_len) < 0)
+    if (bind(listen_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0)
     {
-	DEBUGLOG("Can't bind to port\n");
+	DEBUGLOG("Can't bind to port: %s\n", strerror(errno));
 	syslog(LOG_ERR, "Can't bind to port %d, is clvmd already running ?", tcp_port);
 	close(listen_fd);
 	return -1;
@@ -142,7 +119,7 @@ int alloc_client(int fd, char *csid, struct local_client **new_client)
 {
     struct local_client *client;
 
-    DEBUGLOG("alloc_client %d csid = [%d.%d.%d.%d]\n", fd,csid[0],csid[1],csid[2],csid[3]);
+    DEBUGLOG("alloc_client %d csid = %s\n", fd, print_csid(csid));
 
     /* Create a local_client and return it */
     client = malloc(sizeof(struct local_client));
@@ -199,7 +176,7 @@ int cluster_fd_callback(struct local_client *fd, char *buf, int len, char *csid,
 			struct local_client **new_client)
 {
     int newfd;
-    struct sockaddr_in addr;
+    struct sockaddr_in6 addr;
     socklen_t addrlen = sizeof(addr);
     int status;
     char name[MAX_CLUSTER_MEMBER_NAME_LEN];
@@ -218,22 +195,20 @@ int cluster_fd_callback(struct local_client *fd, char *buf, int len, char *csid,
 
     /* Check that the client is a member of the cluster
        and reject if not.
-       // FIXME: IPv4 specific
     */
-    if (name_from_csid((char *)&addr.sin_addr.s_addr, name) < 0)
+    if (name_from_csid((char *)&addr.sin6_addr, name) < 0)
     {
-	char *ip = (char *)&addr.sin_addr.s_addr;
+	syslog(LOG_ERR, "Got connect from non-cluster node %s\n",
-	syslog(LOG_ERR, "Got connect from non-cluster node %d.%d.%d.%d\n",
+	       print_csid((char *)&addr.sin6_addr));
-	       ip[0], ip[1], ip[2], ip[3]);
+	DEBUGLOG("Got connect from non-cluster node %s\n",
-	DEBUGLOG("Got connect from non-cluster node %d.%d.%d.%d\n",
+		 print_csid((char *)&addr.sin6_addr));
-		 ip[0], ip[1], ip[2], ip[3]);
 	close(newfd);
 
 	errno = EAGAIN;
 	return -1;
     }
 
-    status = alloc_client(newfd, (char *)&addr.sin_addr.s_addr, new_client);
+    status = alloc_client(newfd, (char *)&addr.sin6_addr, new_client);
     if (status)
     {
 	DEBUGLOG("cluster_fd_callback, alloc_client failed, status = %d\n", status);
@@ -250,7 +225,7 @@ int cluster_fd_callback(struct local_client *fd, char *buf, int len, char *csid,
 static int read_from_tcpsock(struct local_client *client, char *buf, int len, char *csid,
 			     struct local_client **new_client)
 {
-    struct sockaddr_in addr;
+    struct sockaddr_in6 addr;
     socklen_t slen = sizeof(addr);
     int status;
 
@@ -259,7 +234,7 @@ static int read_from_tcpsock(struct local_client *client, char *buf, int len, ch
 
     /* Get "csid" */
     getpeername(client->fd, (struct sockaddr *)&addr, &slen);
-    memcpy(csid, &addr.sin_addr.s_addr, MAX_CSID_LEN);
+    memcpy(csid, &addr.sin6_addr, MAX_CSID_LEN);
 
     status = read(client->fd, buf, len);
 
@@ -289,11 +264,11 @@ static int read_from_tcpsock(struct local_client *client, char *buf, int len, ch
 static int connect_csid(char *csid, struct local_client **newclient)
 {
     int fd;
-    struct sockaddr_in addr;
+    struct sockaddr_in6 addr;
     int status;
 
     DEBUGLOG("Connecting socket\n");
-    fd = socket(PF_INET, SOCK_STREAM, 0);
+    fd = socket(PF_INET6, SOCK_STREAM, 0);
 
     if (fd < 0)
     {
@@ -301,12 +276,12 @@ static int connect_csid(char *csid, struct local_client **newclient)
 	return -1;
     }
 
-    addr.sin_family = AF_INET;
+    addr.sin6_family = AF_INET6;
-    memcpy(&addr.sin_addr.s_addr, csid, MAX_CSID_LEN);
+    memcpy(&addr.sin6_addr, csid, MAX_CSID_LEN);
-    addr.sin_port = htons(tcp_port);
+    addr.sin6_port = htons(tcp_port);
 
     DEBUGLOG("Connecting socket %d\n", fd);
-    if (connect(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) < 0)
+    if (connect(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in6)) < 0)
     {
 	syslog(LOG_ERR, "Unable to connect to remote node: %m");
 	DEBUGLOG("Unable to connect to remote node: %s\n", strerror(errno));
@@ -334,7 +309,7 @@ static int tcp_send_message(void *buf, int msglen, unsigned char *csid, const ch
 
     assert(csid);
 
-    DEBUGLOG("tcp_send_message, csid = [%d.%d.%d.%d], msglen = %d\n", csid[0],csid[1],csid[2],csid[3], msglen);
+    DEBUGLOG("tcp_send_message, csid = %s, msglen = %d\n", print_csid(csid), msglen);
 
     /* Don't connect to ourself */
     get_our_csid(ourcsid);
@@ -383,55 +358,17 @@ int cluster_send_message(void *buf, int msglen, char *csid, const char *errtext)
     return status;
 }
 
-static int get_tcp_port(int default_port)
-{
-    int ccs_handle;
-    int port = default_port;
-    char *portstr;
-
-    ccs_handle = ccs_connect();
-    if (ccs_handle)
-    {
-	return port;
-    }
-
-    if (!ccs_get(ccs_handle, "//clvm/@port", &portstr))
-    {
-	port = atoi(portstr);
-	free(portstr);
-
-	if (port <= 0 && port >= 65536)
-	    port = default_port;
-    }
-    ccs_disconnect(ccs_handle);
-
-    DEBUGLOG("Using port %d for communications\n", port);
-    return port;
-}
-
 /* To get our own IP address we get the locally bound address of the
    socket that's talking to GULM in the assumption(eek) that it will
    be on the "right" network in a multi-homed system */
 static int get_our_ip_address(char *addr, int *family)
 {
-    /* Use a sockaddr_in6 to make sure it's big enough */
+	struct utsname info;
-    struct sockaddr_in6 saddr;
+
-    int socklen = sizeof(saddr);
+	uname(&info);
+	get_ip_address(info.nodename, addr);
 
-    if (!getsockname(gulm_fd(), (struct sockaddr *)&saddr, &socklen))
-    {
-	if (saddr.sin6_family == AF_INET6)
-	{
-	    memcpy(addr, &saddr.sin6_addr, sizeof(saddr.sin6_addr));
-	}
-	else
-	{
-	    struct sockaddr_in *sin4 = (struct sockaddr_in *)&saddr;
-	    memcpy(addr, &sin4->sin_addr, sizeof(sin4->sin_addr));
-	}
 	return 0;
-    }
-    return -1;
 }
 
 /* Public version of above for those that don't care what protocol
@@ -454,6 +391,14 @@ void get_our_csid(char *csid)
     memcpy(csid, our_csid, MAX_CSID_LEN);
 }
 
+static void map_v4_to_v6(struct in_addr *ip4, struct in6_addr *ip6)
+{
+   ip6->s6_addr32[0] = 0;
+   ip6->s6_addr32[1] = 0;
+   ip6->s6_addr32[2] = htonl(0xffff);
+   ip6->s6_addr32[3] = ip4->s_addr;
+}
+
 /* Get someone else's IP address from DNS */
 int get_ip_address(char *node, char *addr)
 {
@@ -467,14 +412,29 @@ int get_ip_address(char *node, char *addr)
     /* Try IPv6 first. The man page for gethostbyname implies that
        it will lookup ip6 & ip4 names, but it seems not to */
     he = gethostbyname2(node, AF_INET6);
-    if (!he)
+    if (he)
+    {
+	memcpy(addr, he->h_addr_list[0],
+	       he->h_length);
+    }
+    else
+    {
 	he = gethostbyname2(node, AF_INET);
-    if (!he)
+	if (!he)
-	return -1;
+	    return -1;
-
+	map_v4_to_v6((struct in_addr *)he->h_addr_list[0], (struct in6_addr *)addr);
-    /* For IPv4 address just use the lower 4 bytes */
+    }
-    memcpy(&addr, he->h_addr_list[0],
-	   he->h_length);
 
     return 0;
 }
+
+char *print_csid(char *csid)
+{
+    static char buf[128];
+    int *icsid = (int *)csid;
+
+    sprintf(buf, "[%x.%x.%x.%x]",
+	    icsid[0],icsid[1],icsid[2],icsid[3]);
+
+    return buf;
+}

daemons/clvmd/tcp-comms.h

@@ -4,4 +4,5 @@
 #define MAX_CSID_LEN sizeof(struct in6_addr)
 #define MAX_CLUSTER_MEMBER_NAME_LEN 128
 
-extern int init_comms(void);
+extern int init_comms(unsigned short);
+extern char *print_csid(char *);