smbd: Implement a cleanup daemon

We do way too much stuff in the parent smbd in remove_child_pid(). In
particular accessing ctdbd is not a good idea when ctdbd is stuck in something.
We've had a case where smbd exited itself with "ctdb timeout" being set to 60
seconds. ctdb was just stuck doing recoveries, and the parent smbd was sitting
in serverid_exists trying to retrieve a record for a child that had exited. Not
good.

This daemon sits there as parent->cleanupd and receives MSG_SMB_NOTIFY_CLEANUP
messages that hold the serverid and exit status of a former child. The next
commits will step by step empty remove_child_pid in the parent and move the
tasks to the helper.

Signed-off-by: Volker Lendecke <vl@samba.org>
Reviewed-by: Ralph Boehme <slow@samba.org>

source3/smbd/server.c

@@ -49,6 +49,8 @@
 #include "scavenger.h"
 #include "locking/leases_db.h"
 #include "smbd/notifyd/notifyd.h"
+#include "smbd/smbd_cleanupd.h"
+#include "lib/util/sys_rw.h"
 
 #ifdef CLUSTER_SUPPORT
 #include "ctdb_protocol.h"
@@ -70,6 +72,8 @@ struct smbd_parent_context {
 	struct smbd_child_pid *children;
 	size_t num_children;
 
+	struct server_id cleanupd;
+
 	struct tevent_timer *cleanup_te;
 };
 
@@ -405,6 +409,118 @@ static bool smbd_notifyd_init(struct messaging_context *msg, bool interactive)
 	return tevent_req_poll(req, ev);
 }
 
+static void cleanupd_stopped(struct tevent_req *req);
+
+static bool cleanupd_init(struct messaging_context *msg, bool interactive,
+			  struct server_id *ppid)
+{
+	struct tevent_context *ev = messaging_tevent_context(msg);
+	struct server_id parent_id = messaging_server_id(msg);
+	struct tevent_req *req;
+	pid_t pid;
+	NTSTATUS status;
+	ssize_t rwret;
+	int ret;
+	bool ok;
+	char c;
+	int up_pipe[2];
+
+	if (interactive) {
+		req = smbd_cleanupd_send(msg, ev, msg, parent_id.pid);
+		*ppid = messaging_server_id(msg);
+		return (req != NULL);
+	}
+
+	ret = pipe(up_pipe);
+	if (ret == -1) {
+		DBG_WARNING("pipe failed: %s\n", strerror(errno));
+		return false;
+	}
+
+	pid = fork();
+	if (pid == -1) {
+		DBG_WARNING("fork failed: %s\n", strerror(errno));
+		close(up_pipe[0]);
+		close(up_pipe[1]);
+		return false;
+	}
+
+	if (pid != 0) {
+
+		close(up_pipe[1]);
+		rwret = sys_read(up_pipe[0], &c, 1);
+		close(up_pipe[0]);
+
+		if (rwret == -1) {
+			DBG_WARNING("sys_read failed: %s\n", strerror(errno));
+			return false;
+		}
+		if (rwret == 0) {
+			DBG_WARNING("cleanupd could not start\n");
+			return false;
+		}
+		if (c != 0) {
+			DBG_WARNING("cleanupd returned %d\n", (int)c);
+			return false;
+		}
+
+		DBG_DEBUG("Started cleanupd pid=%d\n", (int)pid);
+
+		*ppid = pid_to_procid(pid);
+		return true;
+	}
+
+	close(up_pipe[0]);
+
+	status = reinit_after_fork(msg, ev, true, "cleanupd");
+	if (!NT_STATUS_IS_OK(status)) {
+		DBG_WARNING("reinit_after_fork failed: %s\n",
+			    nt_errstr(status));
+		c = 1;
+		sys_write(up_pipe[1], &c, 1);
+
+		exit(1);
+	}
+
+	req = smbd_cleanupd_send(msg, ev, msg, parent_id.pid);
+	if (req == NULL) {
+		DBG_WARNING("smbd_cleanupd_send failed\n");
+		c = 2;
+		sys_write(up_pipe[1], &c, 1);
+
+		exit(1);
+	}
+
+	tevent_req_set_callback(req, cleanupd_stopped, msg);
+
+	c = 0;
+	rwret = sys_write(up_pipe[1], &c, 1);
+	close(up_pipe[1]);
+
+	if (rwret == -1) {
+		DBG_WARNING("sys_write failed: %s\n", strerror(errno));
+		exit(1);
+	}
+	if (rwret != 1) {
+		DBG_WARNING("sys_write could not write result\n");
+		exit(1);
+	}
+
+	ok = tevent_req_poll(req, ev);
+	if (!ok) {
+		DBG_WARNING("tevent_req_poll returned %s\n", strerror(errno));
+	}
+	exit(0);
+}
+
+static void cleanupd_stopped(struct tevent_req *req)
+{
+	NTSTATUS status;
+
+	status = smbd_cleanupd_recv(req);
+	DBG_WARNING("cleanupd stopped: %s\n", nt_errstr(status));
+}
+
 /*
   at most every smbd:cleanuptime seconds (default 20), we scan the BRL
   and locking database for entries to cleanup. As a side effect this
@@ -440,10 +556,22 @@ static void remove_child_pid(struct smbd_parent_context *parent,
 {
 	struct smbd_child_pid *child;
 	struct server_id child_id;
+	struct iovec iov[2];
+	NTSTATUS status;
 	int ret;
 
 	child_id = pid_to_procid(pid);
 
+	iov[0] = (struct iovec) { .iov_base = (uint8_t *)&pid,
+				  .iov_len = sizeof(pid) };
+	iov[1] = (struct iovec) { .iov_base = (uint8_t *)&unclean_shutdown,
+				  .iov_len = sizeof(bool) };
+
+	status = messaging_send_iov(parent->msg_ctx, parent->cleanupd,
+				    MSG_SMB_NOTIFY_CLEANUP,
+				    iov, ARRAY_SIZE(iov), NULL, 0);
+	DEBUG(10, ("messaging_send_iov returned %s\n", nt_errstr(status)));
+
 	ret = messaging_cleanup(parent->msg_ctx, pid);
 
 	if ((ret != 0) && (ret != ENOENT)) {
@@ -1476,6 +1604,10 @@ extern void build_options(bool screen);
 		exit_daemon("Samba cannot init notification", EACCES);
 	}
 
+	if (!cleanupd_init(msg_ctx, interactive, &parent->cleanupd)) {
+		exit_daemon("Samba cannot init the cleanupd", EACCES);
+	}
+
 	if (!messaging_parent_dgm_cleanup_init(msg_ctx)) {
 		exit(1);
 	}

source3/smbd/smbd_cleanupd.c

@@ -0,0 +1,106 @@
+/*
+ * Unix SMB/CIFS implementation.
+ *
+ * Copyright (C) Volker Lendecke 2015
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "replace.h"
+#include "smbd_cleanupd.h"
+#include "lib/util/tevent_ntstatus.h"
+#include "lib/util/debug.h"
+
+struct smbd_cleanupd_state {
+	pid_t parent_pid;
+};
+
+static void smbd_cleanupd_shutdown(struct messaging_context *msg,
+				   void *private_data, uint32_t msg_type,
+				   struct server_id server_id,
+				   DATA_BLOB *data);
+static void smbd_cleanupd_process_exited(struct messaging_context *msg,
+					 void *private_data, uint32_t msg_type,
+					 struct server_id server_id,
+					 DATA_BLOB *data);
+
+struct tevent_req *smbd_cleanupd_send(TALLOC_CTX *mem_ctx,
+				      struct tevent_context *ev,
+				      struct messaging_context *msg,
+				      pid_t parent_pid)
+{
+	struct tevent_req *req;
+	struct smbd_cleanupd_state *state;
+	NTSTATUS status;
+
+	req = tevent_req_create(mem_ctx, &state, struct smbd_cleanupd_state);
+	if (req == NULL) {
+		return NULL;
+	}
+	state->parent_pid = parent_pid;
+
+	status = messaging_register(msg, req, MSG_SHUTDOWN,
+				    smbd_cleanupd_shutdown);
+	if (tevent_req_nterror(req, status)) {
+		return tevent_req_post(req, ev);
+	}
+
+	status = messaging_register(msg, req, MSG_SMB_NOTIFY_CLEANUP,
+				    smbd_cleanupd_process_exited);
+	if (tevent_req_nterror(req, status)) {
+		return tevent_req_post(req, ev);
+	}
+
+	return req;
+}
+
+static void smbd_cleanupd_shutdown(struct messaging_context *msg,
+				   void *private_data, uint32_t msg_type,
+				   struct server_id server_id,
+				   DATA_BLOB *data)
+{
+	struct tevent_req *req = talloc_get_type_abort(
+		private_data, struct tevent_req);
+	tevent_req_done(req);
+}
+
+static void smbd_cleanupd_process_exited(struct messaging_context *msg,
+					 void *private_data, uint32_t msg_type,
+					 struct server_id server_id,
+					 DATA_BLOB *data)
+{
+	struct tevent_req *req = talloc_get_type_abort(
+		private_data, struct tevent_req);
+	struct smbd_cleanupd_state *state = tevent_req_data(
+		req, struct smbd_cleanupd_state);
+	pid_t pid;
+	bool unclean_shutdown;
+
+	if (data->length != (sizeof(pid) + sizeof(unclean_shutdown))) {
+		DBG_WARNING("Got invalid length: %zu\n", data->length);
+		return;
+	}
+
+	memcpy(&pid, data->data, sizeof(pid));
+	memcpy(&unclean_shutdown, data->data + sizeof(pid),
+	       sizeof(unclean_shutdown));
+
+	DBG_DEBUG("%d exited %sclean\n", (int)pid,
+		  unclean_shutdown ? "un" : "");
+}
+
+NTSTATUS smbd_cleanupd_recv(struct tevent_req *req)
+{
+	return tevent_req_simple_recv_ntstatus(req);
+}

source3/smbd/smbd_cleanupd.h

@@ -0,0 +1,33 @@
+/*
+ * Unix SMB/CIFS implementation.
+ *
+ * Copyright (C) Volker Lendecke 2014
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __SMBD_CLEANUPD_H__
+#define __SMBD_CLEANUPD_H__
+
+#include "replace.h"
+#include <tevent.h>
+#include "messages.h"
+
+struct tevent_req *smbd_cleanupd_send(TALLOC_CTX *mem_ctx,
+				      struct tevent_context *ev,
+				      struct messaging_context *msg,
+				      pid_t parent_pid);
+NTSTATUS smbd_cleanupd_recv(struct tevent_req *req);
+
+#endif

source3/wscript_build

@@ -858,7 +858,7 @@ bld.SAMBA3_SUBSYSTEM('LIBLSA',
 ########################## BINARIES #################################
 
 bld.SAMBA3_BINARY('smbd/smbd',
-                 source='smbd/server.c',
+                 source='smbd/server.c smbd/smbd_cleanupd.c',
                  deps='smbd_base EPMD LSASD FSSD MDSSD',
                  install_path='${SBINDIR}')