merge from ronnie

(This used to be ctdb commit f67a4842e7b1efb2ad61c41e4895c7698e564bf3)

ctdb/Makefile.in

@@ -37,7 +37,7 @@ CTDB_OBJ = $(CTDB_COMMON_OBJ) $(CTDB_TCP_OBJ) $(POPT_OBJ)
 
 OBJS = @TDB_OBJ@ @TALLOC_OBJ@ @LIBREPLACEOBJ@ @INFINIBAND_WRAPPER_OBJ@ $(EXTRA_OBJ) @EVENTS_OBJ@ $(CTDB_OBJ) $(UTIL_OBJ)
 
-BINS = bin/ctdbd bin/ctdbd_test bin/ctdb_test bin/ctdb_bench bin/ctdb_messaging bin/ctdb_fetch bin/ctdb_fetch1 bin/lockwait bin/ctdb_control bin/ctdb_dump bin/recoverd @INFINIBAND_BINS@
+BINS = bin/ctdbd bin/ctdbd_test bin/ctdb_test bin/ctdb_bench bin/ctdb_messaging bin/ctdb_fetch bin/ctdb_fetch1 bin/lockwait bin/ctdb_control bin/ctdb_dump bin/ctdb_recoverd @INFINIBAND_BINS@
 
 DIRS = lib bin
 
@@ -72,9 +72,9 @@ bin/ctdb_dump: $(OBJS) tools/ctdb_dump.o
 	@echo Linking $@
 	@$(CC) $(CFLAGS) -o $@ tools/ctdb_dump.o $(OBJS) $(LIB_FLAGS)
 
-bin/recoverd: $(OBJS) direct/recoverd.o 
+bin/ctdb_recoverd: $(OBJS) direct/ctdb_recoverd.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ direct/recoverd.o $(OBJS) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ direct/ctdb_recoverd.o $(OBJS) $(LIB_FLAGS)
 
 bin/ctdbd_test: $(OBJS) direct/ctdbd_test.o
 	@echo Linking $@

ctdb/common/cmdline.c

@@ -38,6 +38,7 @@ static struct {
 	int torture;
 	const char *logfile;
 	const char *events;
+	int recovery_daemon;
 } ctdb_cmdline = {
 	.nlist = NULL,
 	.transport = "tcp",
@@ -47,6 +48,7 @@ static struct {
 	.db_dir = NULL,
 	.torture = 0,
 	.logfile = NULL,
+	.recovery_daemon = 0,
 };
 
 enum {OPT_EVENTSYSTEM=1};
@@ -71,6 +73,7 @@ struct poptOption popt_ctdb_cmdline[] = {
 	{ "socket", 0, POPT_ARG_STRING, &ctdb_cmdline.socketname, 0, "local socket name", "filename" },
 	{ "transport", 0, POPT_ARG_STRING, &ctdb_cmdline.transport, 0, "protocol transport", NULL },
 	{ "self-connect", 0, POPT_ARG_NONE, &ctdb_cmdline.self_connect, 0, "enable self connect", "boolean" },
+	{ "recovery-daemon", 0, POPT_ARG_NONE, &ctdb_cmdline.recovery_daemon, 0, "enable recovery daemon", "boolean" },
 	{ "debug", 'd', POPT_ARG_INT, &LogLevel, 0, "debug level"},
 	{ "dbdir", 0, POPT_ARG_STRING, &ctdb_cmdline.db_dir, 0, "directory for the tdb files", NULL },
 	{ "torture", 0, POPT_ARG_NONE, &ctdb_cmdline.torture, 0, "enable nastiness in library", NULL },
@@ -112,6 +115,9 @@ struct ctdb_context *ctdb_cmdline_init(struct event_context *ev)
 	if (ctdb_cmdline.torture) {
 		ctdb_set_flags(ctdb, CTDB_FLAG_TORTURE);
 	}
+	if (ctdb_cmdline.recovery_daemon) {
+		ctdb_set_flags(ctdb, CTDB_FLAG_RECOVERY);
+	}
 
 	ret = ctdb_set_transport(ctdb, ctdb_cmdline.transport);
 	if (ret == -1) {

ctdb/common/ctdb_control.c

@@ -92,6 +92,7 @@ static int traverse_getkeys(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data
 	len+=4; /*lmaster*/
 	len+=4; /*key len*/
 	len+=4; /*data len */
+	len=(len+CTDB_DS_ALIGNMENT-1)& ~(CTDB_DS_ALIGNMENT-1);
 	len+=key.dsize;
 	len=(len+CTDB_DS_ALIGNMENT-1)& ~(CTDB_DS_ALIGNMENT-1);
 	len+=sizeof(struct ctdb_ltdb_header);
@@ -122,6 +123,7 @@ static int traverse_getkeys(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data
 
 	ptr = outdata->dptr+(((ptr-outdata->dptr)+CTDB_DS_ALIGNMENT-1)& ~(CTDB_DS_ALIGNMENT-1));
 	memcpy(ptr, data.dptr+sizeof(struct ctdb_ltdb_header), data.dsize-sizeof(struct ctdb_ltdb_header));
+	ptr = outdata->dptr+(((ptr-outdata->dptr)+CTDB_DS_ALIGNMENT-1)& ~(CTDB_DS_ALIGNMENT-1));
 
 	return 0;
 }

ctdb/common/ctdb_daemon.c

@@ -720,6 +720,15 @@ int ctdb_start_daemon(struct ctdb_context *ctdb)
 	ctdb->ev = event_context_init(NULL);
 	fde = event_add_fd(ctdb->ev, ctdb, ctdb->daemon.sd, EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
 			   ctdb_accept_client, ctdb);
+
+	/* start the recovery daemon */
+	if (ctdb->flags & CTDB_FLAG_RECOVERY) {
+		char cmdstr[256];
+
+		sprintf(cmdstr, "ctdb_recoverd --socket=%s &",domain_socket_name); 
+		system(cmdstr);
+	}
+
 	ctdb_main_loop(ctdb);
 
 	return 0;

ctdb/direct/ctdb_recoverd.c

@@ -378,21 +378,21 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
 
 
 
-	/* pull all remote databases onto the local node */
-	ret = pull_all_remote_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
+	/* repoint all local and remote database records to an invalid
+	   node as being dmaster to stop the shortcut from working
+	 */
+	ret = update_dmaster_on_all_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
 	if (ret != 0) {
-		DEBUG(0, (__location__ "Unable to pull remote databases\n"));
+		DEBUG(0, (__location__ "Unable to update dmaster on all databases\n"));
 		return -1;
 	}
 
 
 
-	/* repoint all local and remote database records to the local
-	   node as being dmaster
-	 */
-	ret = update_dmaster_on_all_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
+	/* pull all remote databases onto the local node */
+	ret = pull_all_remote_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
 	if (ret != 0) {
-		DEBUG(0, (__location__ "Unable to update dmaster on all databases\n"));
+		DEBUG(0, (__location__ "Unable to pull remote databases\n"));
 		return -1;
 	}
 
@@ -439,6 +439,17 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
 	}
 
 
+	/* repoint all local and remote database records to the local
+	   node as being dmaster
+	 */
+	ret = update_dmaster_on_all_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
+	if (ret != 0) {
+		DEBUG(0, (__location__ "Unable to update dmaster on all databases\n"));
+		return -1;
+	}
+
+
+
 	/* disable recovery mode */
 	ret = set_recovery_mode(ctdb, nodemap, CTDB_RECOVERY_NORMAL);
 	if (ret!=0) {

ctdb/include/ctdb.h

@@ -51,6 +51,7 @@ struct ctdb_call_info {
 */
 #define CTDB_FLAG_SELF_CONNECT (1<<0)
 #define CTDB_FLAG_TORTURE      (1<<1)
+#define CTDB_FLAG_RECOVERY     (1<<2)
 
 
 /* 

ctdb/tests/recover.sh

@@ -1,15 +1,13 @@
 #!/bin/sh
 
 killall -q ctdbd
-killall -q recoverd
+killall -q ctdb_recoverd
 
 echo "Starting 4 ctdb daemons"
-bin/ctdbd --nlist direct/4nodes.txt
-bin/ctdbd --nlist direct/4nodes.txt --listen=127.0.0.2:9001 --socket=/tmp/ctdb.socket.127.0.0.2
-bin/ctdbd --nlist direct/4nodes.txt --listen=127.0.0.3:9001 --socket=/tmp/ctdb.socket.127.0.0.3
-bin/ctdbd --nlist direct/4nodes.txt --listen=127.0.0.4:9001 --socket=/tmp/ctdb.socket.127.0.0.4
-echo "Starting one recovery daemon on node 0"
-bin/recoverd --socket=/tmp/ctdb.socket >/dev/null 2>/dev/null &
+bin/ctdbd --recovery-daemon --nlist direct/4nodes.txt
+bin/ctdbd --recovery-daemon --nlist direct/4nodes.txt --listen=127.0.0.2:9001 --socket=/tmp/ctdb.socket.127.0.0.2
+bin/ctdbd --recovery-daemon --nlist direct/4nodes.txt --listen=127.0.0.3:9001 --socket=/tmp/ctdb.socket.127.0.0.3
+bin/ctdbd --recovery-daemon --nlist direct/4nodes.txt --listen=127.0.0.4:9001 --socket=/tmp/ctdb.socket.127.0.0.4
 
 echo
 echo "Attaching to some databases"

ctdb/tools/monitor_recovery.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+CTDB_CONTROL=./bin/ctdb_control
+XPOS=0
+
+$CTDB_CONTROL getnodemap 0 | egrep "^vnn:" | sed -e "s/^vnn://" -e "s/ .*$//" | while read NODE; do
+	xterm -geometry 30x25+$XPOS -e "while true; do sleep 1; clear; $CTDB_CONTROL getnodemap $NODE; $CTDB_CONTROL getvnnmap $NODE; $CTDB_CONTROL getrecmode $NODE; $CTDB_CONTROL getrecmaster $NODE;done" &
+	export XPOS=`expr $XPOS "+" "200"`
+
+done
+
+
+