S3: Detect max_open_files from system

- Attempt to use syscalls to determine max-open-files value.
- Add in periodic logging when max file limit reached

source3/configure.in

@@ -836,6 +836,25 @@ if test x"$samba_cv_msghdr_msg_control" = x"yes"; then
 	     [If we can use msg_control for passing file descriptors])
 fi
 
+#############################################
+# check for sysctlbyname
+AC_CACHE_CHECK([for sysctlbyname],samba_cv_sysctlbyname, [
+    AC_TRY_COMPILE([
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <stdlib.h>
+#include <stddef.h>],
+[
+	int sysctl_max;
+	size_t size = sizeof(sysctl_max);
+	sysctlbyname("test", &sysctl_max, &size, NULL, 0);
+],
+	samba_cv_sysctlbyname=yes,samba_cv_sysctlbyname=no)])
+if test x"$samba_cv_msghdr_msg_control" = x"yes"; then
+    AC_DEFINE(HAVE_SYSCTLBYNAME,1,
+	     [If we support sysctlbyname api])
+fi
+
 #############################################
 # check for fd passing struct via msg_acctrights
 AC_CACHE_CHECK([for fd passing via msg_acctrights],

source3/param/loadparm.c

@@ -4657,6 +4657,42 @@ static void init_printer_values(struct service *pService)
 
 	}
 }
+/**
+ *  Function to return the default value for the maximum number of open
+ *  file descriptors permitted.  This function tries to consult the
+ *  kernel-level (sysctl) and ulimit (getrlimit()) values and goes
+ *  the smaller of those.
+ */
+static int max_open_files(void)
+{
+	int sysctl_max;
+	struct rlimit rl;
+	bool sysctl_worked = false, rlimit_worked = false;
+
+#ifdef HAVE_SYSCTLBYNAME
+	size_t size = sizeof(sysctl_max);
+	if (sysctlbyname("kern.maxfilesperproc", &sysctl_max, &size, NULL,0)==0)
+		sysctl_worked = true;
+#endif
+
+	if (getrlimit(RLIMIT_NOFILE, &rl) == 0)
+		rlimit_worked = true;
+
+	if (sysctl_worked) {
+		if ((!rlimit_worked) ||
+		    (rl.rlim_cur == RLIM_INFINITY) ||
+		    (rl.rlim_cur > sysctl_max))
+			return sysctl_max;
+		else
+			return rl.rlim_cur;
+	} else {
+		if ((!rlimit_worked) ||
+		    (rl.rlim_cur == RLIM_INFINITY))
+			return MAX_OPEN_FILES;
+		else
+			return rl.rlim_cur;
+	}
+}
 
 /**
  * Common part of freeing allocated data for one parameter.
@@ -4880,7 +4916,7 @@ static void init_globals(bool first_time_only)
 	Globals.getwd_cache = true;
 	Globals.bLargeReadwrite = True;
 	Globals.max_log_size = 5000;
-	Globals.max_open_files = MAX_OPEN_FILES;
+	Globals.max_open_files = max_open_files();
 	Globals.open_files_db_hash_size = SMB_OPEN_DATABASE_TDB_HASH_SIZE;
 	Globals.maxprotocol = PROTOCOL_NT1;
 	Globals.minprotocol = PROTOCOL_CORE;

source3/smbd/open.c

@@ -132,6 +132,18 @@ static NTSTATUS fd_open(struct connection_struct *conn,
 	fsp->fh->fd = SMB_VFS_OPEN(conn,fname,fsp,flags,mode);
 	if (fsp->fh->fd == -1) {
 		status = map_nt_error_from_unix(errno);
+		if (errno == EMFILE) {
+			static time_t last_warned = 0L;
+
+			if (time((time_t *) NULL) > last_warned) {
+				DEBUG(0,("Too many open files, unable "
+					"to open more!  smbd's max "
+					"open files = %d\n",
+					lp_max_open_files()));
+				last_warned = time((time_t *) NULL);
+			}
+		}
+
 	}
 
 	DEBUG(10,("fd_open: name %s, flags = 0%o mode = 0%o, fd = %d. %s\n",