Don't use sysconf(_SC_NGROUPS_MAX) on macOS for getgroups()

On MacOS sysconf(_SC_NGROUPS_MAX) always returns 16. However, this is not
the value used by getgroups(2). MacOS uses nested groups but getgroups(2)
will return the flattened list which can easily exceed 16 groups. In my
testing getgroups() already returns 16 groups on a freshly installed
system. And on a 10.14 system the root user is in more than 16 groups by
default which makes it impossible to run smbd without this change.
Setting _DARWIN_UNLIMITED_GETGROUPS allows getgroups() to return more than
16 groups. This also changes set_unix_security_ctx() to only set up to
16 groups since that is the limit for initgroups() according to the manpage.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=8773

Signed-off-by: Alex Richardson <Alexander.Richardson@cl.cam.ac.uk>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Reviewed-by: Jeremy Allison <jra@samba.org>

Autobuild-User(master): Jeremy Allison <jra@samba.org>
Autobuild-Date(master): Thu Sep  9 17:43:19 UTC 2021 on sn-devel-184

buildtools/wafsamba/wscript

@@ -539,7 +539,10 @@ struct foo bar = { .y = 'X', .x = 1 };
     conf.CHECK_HEADERS('strings.h inttypes.h stdint.h unistd.h minix/config.h', add_headers=True)
     conf.CHECK_HEADERS('ctype.h', add_headers=True)
 
-    if sys.platform != 'darwin':
+    if sys.platform == 'darwin':
+        conf.DEFINE('_DARWIN_C_SOURCE', 1, add_to_cflags=True)
+        conf.DEFINE('_DARWIN_UNLIMITED_GETGROUPS', 1, add_to_cflags=True)
+    else:
         conf.CHECK_HEADERS('standards.h', add_headers=True)
 
     conf.CHECK_HEADERS('stdbool.h stdint.h stdarg.h vararg.h', add_headers=True)

source3/include/proto.h

@@ -230,7 +230,8 @@ void set_effective_capability(enum smbd_capability capability);
 void drop_effective_capability(enum smbd_capability capability);
 long sys_random(void);
 void sys_srandom(unsigned int seed);
-int groups_max(void);
+int getgroups_max(void);
+int setgroups_max(void);
 int sys_getgroups(int setlen, gid_t *gidset);
 int sys_setgroups(gid_t UNUSED(primary_gid), int setlen, gid_t *gidset);
 uint32_t unix_dev_major(SMB_DEV_T dev);

source3/lib/system.c

@@ -756,7 +756,7 @@ void sys_srandom(unsigned int seed)
  Returns equivalent to NGROUPS_MAX - using sysconf if needed.
 ****************************************************************************/
 
-int groups_max(void)
+int setgroups_max(void)
 {
 #if defined(SYSCONF_SC_NGROUPS_MAX)
 	int ret = sysconf(_SC_NGROUPS_MAX);
@@ -766,6 +766,24 @@ int groups_max(void)
 #endif
 }
 
+int getgroups_max(void)
+{
+#if defined(DARWINOS)
+	/*
+	 * On MacOS sysconf(_SC_NGROUPS_MAX) returns 16 due to MacOS's group
+	 * nesting. However, The initgroups() manpage states the following:
+	 * "Note that OS X supports group membership in an unlimited number
+	 * of groups. The OS X kernel uses the group list stored in the process
+	 * credentials only as an initial cache.  Additional group memberships
+	 * are determined by communication between the operating system and the
+	 * opendirectoryd daemon."
+	 */
+	return INT_MAX;
+#else
+	return setgroups_max();
+#endif
+}
+
 /**************************************************************************
  Wrap setgroups and getgroups for systems that declare getgroups() as
  returning an array of gid_t, but actuall return an array of int.
@@ -831,7 +849,7 @@ static int sys_broken_setgroups(int setlen, gid_t *gidset)
 	if (setlen == 0)
 		return 0 ;
 
-	if (setlen < 0 || setlen > groups_max()) {
+	if (setlen < 0 || setlen > setgroups_max()) {
 		errno = EINVAL; 
 		return -1;   
 	}
@@ -882,7 +900,7 @@ static int sys_bsd_setgroups(gid_t primary_gid, int setlen, const gid_t *gidset)
 	int ret;
 
 	/* setgroups(2) will fail with EINVAL if we pass too many groups. */
-	max = groups_max();
+	max = setgroups_max();
 
 	/* No group list, just make sure we are setting the efective GID. */
 	if (setlen == 0) {

source3/lib/system_smbd.c

@@ -205,7 +205,7 @@ bool getgroups_unix_user(TALLOC_CTX *mem_ctx, const char *user,
 			 gid_t primary_gid,
 			 gid_t **ret_groups, uint32_t *p_ngroups)
 {
-	int max_grp = MIN(128, groups_max());
+	int max_grp = MIN(128, getgroups_max());
 	gid_t stack_groups[max_grp];
 	uint32_t ngrp;
 	gid_t *temp_groups = stack_groups;

source3/smbd/sec_ctx.c

@@ -282,7 +282,7 @@ static void set_unix_security_ctx(uid_t uid, gid_t gid, int ngroups, gid_t *grou
 
 static void set_unix_security_ctx(uid_t uid, gid_t gid, int ngroups, gid_t *groups)
 {
-	int max = groups_max();
+	int max = NGROUPS_MAX;
 
 	/* Start context switch */
 	gain_root();

testsuite/smbd/sec_ctx_utils.c

@@ -30,7 +30,7 @@ void get_random_grouplist(int *ngroups, gid_t **groups)
 {
 	int i;
 
-	*ngroups = random() % groups_max();
+	*ngroups = random() % setgroups_max();
 	*groups = malloc(*ngroups * sizeof(gid_t));
 
 	if (!groups) {