samba_dsdb: Use and maintain compatibleFeatures and requiredFeatures in @SAMBA_DSDB

This will allow us to introduce new database features that are
backward compatible from the point of view of older versions of Samba,
but which will be damaged by modifying the database with such a
version.

For example, if linked attributes are stored in sorted order in 4.7,
and this change, without any values in current_supportedFeatures is
itself included in 4.6, then our sortedLinks are backward compatible
to that release.

That is with 4.6 (including this patch) which doesn't care about
ordering -- but a downgraded 4.7 database used by 4.6 will be broken
when later used with 4.7.  If we add a 'sortedLinks' feature flag in
compatibleFeatures, we can detect that.

This will allow us to determine if the database still contains
unsorted links, as that information allows us to make the code
handling links much more efficient.

We won't add the actual flag until all the code is in place.

Andrew wrote the actual code and Douglas wrote the tests, and they
cross-reviewed.

Signed-off-by: Andrew Bartlett <abartlet@samba.org>
Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Piar-programmed-with: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>

selftest: check for database features flags
(cherry picked from commit 8368e06ff65cc70e1cf13a0eb4349033e068fcc6)

BUG 12573: Samba < 4.7 does not know about compatibleFeatures and
requiredFeatures

source4/dsdb/samdb/ldb_modules/samba_dsdb.c

@@ -231,7 +231,7 @@ static int set_ldap_credentials(struct ldb_context *ldb, bool use_external)
 static int samba_dsdb_init(struct ldb_module *module)
 {
 	struct ldb_context *ldb = ldb_module_get_ctx(module);
-	int ret, len, i;
+	int ret, len, i, j;
 	TALLOC_CTX *tmp_ctx = talloc_new(module);
 	struct ldb_result *res;
 	struct ldb_message *rootdse_msg = NULL, *partition_msg;
@@ -317,11 +317,15 @@ static int samba_dsdb_init(struct ldb_module *module)
 	static const char *openldap_backend_modules[] = {
 		"dsdb_flags_ignore", "entryuuid", "simple_dn", NULL };
 
-	static const char *samba_dsdb_attrs[] = { "backendType", NULL };
+	static const char *samba_dsdb_attrs[] = { "backendType",
+						  SAMBA_COMPATIBLE_FEATURES_ATTR,
+						  SAMBA_REQUIRED_FEATURES_ATTR, NULL };
 	static const char *partition_attrs[] = { "ldapBackend", NULL };
 	const char *backendType, *backendUrl;
 	bool use_sasl_external = false;
 
+	const char *current_supportedFeatures[] = {};
+
 	if (!tmp_ctx) {
 		return ldb_oom(ldb);
 	}
@@ -357,7 +361,77 @@ static int samba_dsdb_init(struct ldb_module *module)
 	if (ret == LDB_ERR_NO_SUCH_OBJECT) {
 		backendType = "ldb";
 	} else if (ret == LDB_SUCCESS) {
+		struct ldb_message_element *requiredFeatures;
+		struct ldb_message_element *old_compatibleFeatures;
+
 		backendType = ldb_msg_find_attr_as_string(res->msgs[0], "backendType", "ldb");
+
+		requiredFeatures = ldb_msg_find_element(res->msgs[0], SAMBA_REQUIRED_FEATURES_ATTR);
+		if (requiredFeatures != NULL) {
+			ldb_set_errstring(ldb, "This Samba database was created with "
+					  "a newer Samba version and is marked with "
+					  "requiredFeatures in @SAMBA_DSDB.  "
+					  "This database can not safely be read by this Samba version");
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+
+		old_compatibleFeatures = ldb_msg_find_element(res->msgs[0],
+							      SAMBA_COMPATIBLE_FEATURES_ATTR);
+
+		if (old_compatibleFeatures) {
+			struct ldb_message *features_msg;
+			struct ldb_message_element *features_el;
+
+			features_msg = ldb_msg_new(res);
+			if (features_msg == NULL) {
+				return ldb_module_operr(module);
+			}
+			features_msg->dn = samba_dsdb_dn;
+
+			ldb_msg_add_empty(features_msg, SAMBA_COMPATIBLE_FEATURES_ATTR,
+					  LDB_FLAG_MOD_DELETE, &features_el);
+
+			for (i = 0;
+			     old_compatibleFeatures && i < old_compatibleFeatures->num_values;
+			     i++) {
+				for (j = 0;
+				     j < ARRAY_SIZE(current_supportedFeatures); j++) {
+					if (strcmp((char *)old_compatibleFeatures->values[i].data,
+						   current_supportedFeatures[j]) == 0) {
+						break;
+					}
+				}
+				if (j == ARRAY_SIZE(current_supportedFeatures)) {
+					/*
+					 * Add to list of features to remove
+					 * (rather than all features)
+					 */
+					ret = ldb_msg_add_value(features_msg, SAMBA_COMPATIBLE_FEATURES_ATTR,
+								&old_compatibleFeatures->values[i],
+								NULL);
+					if (ret != LDB_SUCCESS) {
+						return ret;
+					}
+				}
+			}
+			if (features_el->num_values > 0) {
+				/* Delete by list */
+				ret = ldb_next_start_trans(module);
+				if (ret != LDB_SUCCESS) {
+					return ret;
+				}
+				ret = dsdb_module_modify(module, features_msg, DSDB_FLAG_NEXT_MODULE, NULL);
+				if (ret != LDB_SUCCESS) {
+					ldb_next_del_trans(module);
+					return ret;
+				}
+				ret = ldb_next_end_trans(module);
+				if (ret != LDB_SUCCESS) {
+					return ret;
+				}
+			}
+		}
+
 	} else {
 		talloc_free(tmp_ctx);
 		return ret;

source4/dsdb/samdb/samdb.h

@@ -314,4 +314,6 @@ struct dsdb_extended_sec_desc_propagation_op {
  */
 #define DSDB_FLAG_INTERNAL_FORCE_META_DATA 0x10000
 
+#define SAMBA_COMPATIBLE_FEATURES_ATTR "compatibleFeatures"
+#define SAMBA_REQUIRED_FEATURES_ATTR "requiredFeatures"
 #endif /* __SAMDB_H__ */

source4/selftest/tests.py

@@ -625,6 +625,11 @@ plantestsuite_loadlist("samba4.deletetest.python(ad_dc_ntvfs)", "ad_dc_ntvfs", [
 plantestsuite("samba4.blackbox.samba3dump", "none", [os.path.join(samba4srcdir, "selftest/test_samba3dump.sh")])
 plantestsuite("samba4.blackbox.upgrade", "none", ["PYTHON=%s" % python, os.path.join(samba4srcdir, "setup/tests/blackbox_s3upgrade.sh"), '$PREFIX/provision'])
 plantestsuite("samba4.blackbox.provision.py", "none", ["PYTHON=%s" % python, os.path.join(samba4srcdir, "setup/tests/blackbox_provision.sh"), '$PREFIX/provision'])
+plantestsuite("samba4.blackbox.supported_features", "none",
+              ["PYTHON=%s" % python,
+               os.path.join(samba4srcdir,
+                            "setup/tests/blackbox_supported_features.sh"),
+               '$PREFIX/provision'])
 plantestsuite("samba4.blackbox.upgradeprovision.current", "none", ["PYTHON=%s" % python, os.path.join(samba4srcdir, "setup/tests/blackbox_upgradeprovision.sh"), '$PREFIX/provision'])
 plantestsuite("samba4.blackbox.setpassword.py", "none", ["PYTHON=%s" % python, os.path.join(samba4srcdir, "setup/tests/blackbox_setpassword.sh"), '$PREFIX/provision'])
 plantestsuite("samba4.blackbox.newuser.py", "none", ["PYTHON=%s" % python, os.path.join(samba4srcdir, "setup/tests/blackbox_newuser.sh"), '$PREFIX/provision'])

source4/setup/tests/blackbox_supported_features.sh

@@ -0,0 +1,86 @@
+#!/bin/sh
+
+if [ $# -lt 1 ]; then
+cat <<EOF
+Usage: blackbox_supported_features.sh PREFIX
+EOF
+exit 1;
+fi
+
+PREFIX="$1"
+shift 1
+
+DBPATH=$PREFIX/supported-features
+
+. `dirname $0`/../../../testprogs/blackbox/subunit.sh
+
+ldbmodify="ldbmodify"
+if [ -x "$BINDIR/ldbmodify" ]; then
+    ldbmodify="$BINDIR/ldbmodify"
+fi
+
+ldbdel="ldbdel"
+if [ -x "$BINDIR/ldbdel" ]; then
+    ldbdel="$BINDIR/ldbdel"
+fi
+
+ldbsearch="ldbsearch"
+if [ -x "$BINDIR/ldbsearch" ]; then
+    ldbsearch="$BINDIR/ldbsearch"
+fi
+
+testit "provision" $PYTHON $BINDIR/samba-tool domain provision \
+       --domain=FOO --realm=foo.example.com \
+       --targetdir=$DBPATH --use-ntvfs
+
+testit "add-compatible-feature" $ldbmodify \
+       -H tdb://$DBPATH/private/sam.ldb <<EOF
+dn: @SAMBA_DSDB
+changetype: modify
+add: compatibleFeatures
+compatibleFeatures: non-existent-feature
+-
+
+EOF
+
+# The non-existent feature is not compatible with this version, so it
+# should not be listed in compatibleFeatures even though we tried to
+# put it there.
+
+ldb_search_fail() {
+    $ldbsearch -H tdb://$DBPATH/private/sam.ldb \
+               -s base -b "$1" "$2" \
+        |   grep -q "$3"
+}
+
+
+testit_expect_failure "find-compatible-feature" \
+                      ldb_search_fail '@SAMBA_DSDB' 'compatibleFeatures' non-existent-feature
+
+
+# just make sure the thing we're using is normally findable
+testit "find-test-feature" \
+       $ldbsearch -H tdb://$DBPATH/private/sam.ldb \
+       -b 'CN=LostAndFound,DC=foo,DC=example,DC=com'
+
+
+testit "add-required-feature" $ldbmodify \
+       -H tdb://$DBPATH/private/sam.ldb <<EOF
+dn: @SAMBA_DSDB
+changetype: modify
+add: requiredFeatures
+requiredFeatures: futuristic-feature
+-
+
+EOF
+
+# The futuristic-feature is not implemented in this version, but it is
+# required by this database. A search for anything should fail.
+
+testit_expect_failure "find-required-feature" \
+                      $ldbsearch -H tdb://$DBPATH/private/sam.ldb \
+                      -b 'CN=LostAndFound,DC=foo,DC=example,DC=com'
+
+rm -rf $DBPATH
+
+exit $failed