Merge commit 'origin/master'

(This used to be ctdb commit 096cdc0c12d22d99f8405bee5cb9f05c616c8492)

ctdb/Makefile.in

@@ -207,6 +207,7 @@ install: all
 	${INSTALLCMD} -m 644 config/events.d/README $(DESTDIR)$(docdir)/ctdb/README.eventscripts
 	${INSTALLCMD} -m 644 doc/recovery-process.txt $(DESTDIR)$(docdir)/ctdb/recovery-process.txt
 	${INSTALLCMD} -m 755 config/events.d/00.ctdb $(DESTDIR)$(etcdir)/ctdb/events.d
+	${INSTALLCMD} -m 755 config/events.d/01.reclock $(DESTDIR)$(etcdir)/ctdb/events.d
 	${INSTALLCMD} -m 755 config/events.d/10.interface $(DESTDIR)$(etcdir)/ctdb/events.d
 	${INSTALLCMD} -m 755 config/events.d/11.natgw $(DESTDIR)$(etcdir)/ctdb/events.d
 	${INSTALLCMD} -m 755 config/events.d/11.routing $(DESTDIR)$(etcdir)/ctdb/events.d

ctdb/client/ctdb_client.c

@@ -3107,11 +3107,13 @@ static int ctdb_transaction_fetch_start(struct ctdb_transaction_handle *h)
 {
 	struct ctdb_record_handle *rh;
 	TDB_DATA key;
+	TDB_DATA data;
 	struct ctdb_ltdb_header header;
 	TALLOC_CTX *tmp_ctx;
 	const char *keyname = CTDB_TRANSACTION_LOCK_KEY;
 	int ret;
 	struct ctdb_db_context *ctdb_db = h->ctdb_db;
+	pid_t pid;
 
 	key.dptr = discard_const(keyname);
 	key.dsize = strlen(keyname);
@@ -3130,6 +3132,21 @@ again:
 		talloc_free(tmp_ctx);
 		return -1;
 	}
+	/*
+	 * store the pid in the database:
+	 * it is not enough that the node is dmaster...
+	 */
+	pid = getpid();
+	data.dptr = (unsigned char *)&pid;
+	data.dsize = sizeof(pid_t);
+	ret = ctdb_ltdb_store(ctdb_db, key, &(rh->header), data);
+	if (ret != 0) {
+		DEBUG(DEBUG_ERR, (__location__ " Failed to store pid in "
+				  "transaction record\n"));
+		talloc_free(tmp_ctx);
+		return -1;
+	}
+
 	talloc_free(rh);
 
 	ret = tdb_transaction_start(ctdb_db->ltdb->tdb);
@@ -3139,13 +3156,19 @@ again:
 		return -1;
 	}
 
-	ret = ctdb_ltdb_fetch(ctdb_db, key, &header, tmp_ctx, NULL);
+	ret = ctdb_ltdb_fetch(ctdb_db, key, &header, tmp_ctx, &data);
 	if (ret != 0 || header.dmaster != ctdb_db->ctdb->pnn) {
 		tdb_transaction_cancel(ctdb_db->ltdb->tdb);
 		talloc_free(tmp_ctx);
 		goto again;
 	}
 
+	if ((data.dsize != sizeof(pid_t)) || (*(pid_t *)(data.dptr) != pid)) {
+		tdb_transaction_cancel(ctdb_db->ltdb->tdb);
+		talloc_free(tmp_ctx);
+		goto again;
+	}
+
 	talloc_free(tmp_ctx);
 
 	return 0;

ctdb/config/events.d/01.reclock

@@ -0,0 +1,58 @@
+#!/bin/sh
+# script to check accessibility to the reclock file on a node
+
+. $CTDB_BASE/functions
+loadconfig ctdb
+
+cmd="$1"
+shift
+
+PATH=/usr/bin:/bin:/usr/sbin:/sbin:$PATH
+
+# The size of this file represents the number of intervals that have
+# passed when we have tried to but failed to stat the reclock file.
+# after third failure the node becomes unhealthy
+# after the twenteth failure the node we shutdown ctdbd
+RECLOCKCOUNT="$CTDB_BASE/state/reclock-fail-count"
+
+case $cmd in 
+     startup)
+	echo -n > $RECLOCKCOUNT
+	;;
+
+      monitor)
+	echo -n 1 >> $RECLOCKCOUNT
+
+	COUNT=`ls -ln $RECLOCKCOUNT | cut -d" " -f5`
+	[ $COUNT -gt 19 ] && {
+		echo "Reclock file can not be accessed. Shutting down."
+		sleep 1
+		ctdb shutdown
+	}
+
+	RECLOCKFILE=`ctdb -Y getreclock`
+	[ -z $RECLOCKFILE ] && {
+		# we are not using a reclock file
+		echo -n > $RECLOCKCOUNT
+		exit 0
+	}
+
+	# try stat the reclock file as a background process
+	# so that we dont block in case the cluster filesystem is unavailable
+	(
+		stat $RECLOCKFILE
+		[ "$?" -eq 0 ] && {
+			# we could stat the file, reset the counter
+			echo -n > $RECLOCKCOUNT
+		}
+	) >/dev/null 2>/dev/null &
+
+
+	[ $COUNT -gt 2 ] && {
+		echo "Reclock file can not be accessed. Mark node UNHEALTHY."
+		exit 1;
+	}
+	;;
+esac
+
+exit 0

ctdb/packaging/RPM/ctdb.spec

@@ -104,6 +104,7 @@ fi
 %{_docdir}/ctdb/README.eventscripts
 %{_docdir}/ctdb/recovery-process.txt
 %{_sysconfdir}/ctdb/events.d/00.ctdb
+%{_sysconfdir}/ctdb/events.d/01.reclock
 %{_sysconfdir}/ctdb/events.d/10.interface
 %{_sysconfdir}/ctdb/events.d/11.natgw
 %{_sysconfdir}/ctdb/events.d/11.routing

ctdb/tests/src/ctdb_persistent.c

@@ -164,7 +164,7 @@ static void test_store_records(struct ctdb_context *ctdb, struct event_context *
 		ret = ctdb_transaction_commit(h);
 		if (ret != 0) {
 			DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
-			//exit(1);
+			exit(1);
 		}
 
 		/* store the counters and verify that they are sane */

ctdb/tests/src/ctdb_transaction.c

@@ -197,7 +197,7 @@ static void test_store_records(struct ctdb_context *ctdb, struct event_context *
 			ret = ctdb_transaction_commit(h);
 			if (ret != 0) {
 				DEBUG(DEBUG_ERR,("Failed to commit transaction\n"));
-				//exit(1);
+				exit(1);
 			}
 			if (verbose) printf("transaction committed\n");
 		} else {

ctdb/tools/ctdb.c

@@ -2528,6 +2528,11 @@ static int control_getreclock(struct ctdb_context *ctdb, int argc, const char **
 	if (ret != 0) {
 		DEBUG(DEBUG_ERR, ("Unable to get reclock file from node %u\n", options.pnn));
 		return ret;
+	} else {
+		if (options.machinereadable){
+			if (reclock != NULL) {
+				printf("%s", reclock);
+			}
 		} else {
 			if (reclock == NULL) {
 				printf("No reclock file used.\n");
@@ -2535,6 +2540,7 @@ static int control_getreclock(struct ctdb_context *ctdb, int argc, const char **
 				printf("Reclock file:%s\n", reclock);
 			}
 		}
+	}
 	return 0;
 }