domain.py: Add schema upgrade option to samba-tool

Microsoft has published the Schema updates that its Adprep.exe tool
applies when it upgrades a 2008R2 schema to 2012R2.

This patch adds an option to samba-tool to go through these update files
and apply each change one by one. Along the way we need to make a few
changes to the LDIF operations, e.g. change 'ntdsschemaadd' to 'add' and
so on.

The bulk of the changes involve parsing the .ldif file and separating
out each update into a separate operation.

There are a couple of errors that we've chosen to ignore:
- Trying to set isDefunct for an object we don't know about.
- Trying to set a value for an attribute OID that we don't know about
  (we may need to fix this in future, but it'll require some help from
   Microsoft about what the OIDs actually are).

To try to make life easier, I've added a ldif_schema_update helper
class. This provides convenient access of the DN the change applies to
and other such details (whether it's setting isDefunct, etc).

Pair-programmed-with: Garming Sam <garming@catalyst.net.nz>

Signed-off-by: Tim Beale <timbeale@catalyst.net.nz>
Signed-off-by: Garming Sam <garming@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>

python/samba/netcmd/domain.py

@@ -85,7 +85,8 @@ from samba.dsdb import (
 from samba.provision import (
     provision,
     ProvisioningError,
-    DEFAULT_MIN_PWD_LENGTH
+    DEFAULT_MIN_PWD_LENGTH,
+    setup_path
     )
 
 from samba.provision.common import (
@@ -3852,6 +3853,231 @@ class cmd_domain_tombstones(SuperCommand):
     subcommands = {}
     subcommands["expunge"] = cmd_domain_tombstones_expunge()
 
+class ldif_schema_update:
+    """Helper class for applying LDIF schema updates"""
+
+    def __init__(self):
+        self.is_defunct = False
+        self.unknown_oid = None
+        self.dn = None
+        self.ldif = ""
+
+    def _ldap_schemaUpdateNow(self, samdb):
+        ldif = """
+dn:
+changetype: modify
+add: schemaUpdateNow
+schemaUpdateNow: 1
+"""
+        samdb.modify_ldif(ldif)
+
+    def can_ignore_failure(self, error):
+        """Checks if we can safely ignore failure to apply an LDIF update"""
+        (num, errstr) = error.args
+
+        # Microsoft has marked objects as defunct that Samba doesn't know about
+        if num == ldb.ERR_NO_SUCH_OBJECT and self.is_defunct:
+            print("Defunct object %s doesn't exist, skipping" % self.dn)
+            return True
+        elif self.unknown_oid is not None:
+            print("Skipping unknown OID %s for object %s" %(self.unknown_oid, self.dn))
+            return True
+
+        return False
+
+    def apply(self, samdb):
+        """Applies a single LDIF update to the schema"""
+
+        try:
+            samdb.modify_ldif(self.ldif, controls=['relax:0'])
+        except ldb.LdbError as e:
+            if self.can_ignore_failure(e):
+                return 0
+            else:
+                print("Exception: %s" % e)
+                print("Encountered while trying to apply the following LDIF")
+                print("----------------------------------------------------")
+                print("%s" % self.ldif)
+
+                raise
+
+        # REFRESH AFTER EVERY CHANGE
+        # Otherwise the OID-to-attribute mapping in _apply_updates_in_file()
+        # won't work, because it can't lookup the new OID in the schema
+        self._ldap_schemaUpdateNow(samdb)
+
+        return 1
+
+class cmd_domain_schema_upgrade(Command):
+    """Domain schema upgrading"""
+
+    synopsis = "%prog [options]"
+
+    takes_optiongroups = {
+        "sambaopts": options.SambaOptions,
+        "versionopts": options.VersionOptions,
+        "credopts": options.CredentialsOptions,
+    }
+
+    takes_options = [
+        Option("-H", "--URL", help="LDB URL for database or target server", type=str,
+               metavar="URL", dest="H"),
+        Option("--quiet", help="Be quiet", action="store_true"),
+        Option("--verbose", help="Be verbose", action="store_true"),
+        Option("--schema", type="choice", metavar="SCHEMA",
+               choices=["2012", "2012_R2"],
+               help="The schema file to upgrade to. Default is (Windows) 2012_R2.",
+               default="2012_R2")
+    ]
+
+    def _apply_updates_in_file(self, samdb, ldif_file):
+        """
+        Applies a series of updates specified in an .LDIF file. The .LDIF file
+        is based on the adprep Schema updates provided by Microsoft.
+        """
+        count = 0
+        ldif_op = ldif_schema_update()
+
+        # parse the file line by line and work out each update operation to apply
+        for line in ldif_file:
+
+            line = line.rstrip()
+
+            # the operations in the .LDIF file are separated by blank lines. If
+            # we hit a blank line, try to apply the update we've parsed so far
+            if line == '':
+
+                # keep going if we haven't parsed anything yet
+                if ldif_op.ldif == '':
+                    continue
+
+                # Apply the individual change
+                count += ldif_op.apply(samdb)
+
+                # start storing the next operation from scratch again
+                ldif_op = ldif_schema_update()
+                continue
+
+            # replace the placeholder domain name in the .ldif file with the real domain
+            if line.upper().endswith('DC=X'):
+                line = line[:-len('DC=X')] + str(samdb.get_default_basedn())
+            elif line.upper().endswith('CN=X'):
+                line = line[:-len('CN=X')] + str(samdb.get_default_basedn())
+
+            values = line.split(':')
+
+            if values[0].lower() == 'dn':
+                ldif_op.dn = values[1].strip()
+
+            # replace the Windows-specific operation with the Samba one
+            if values[0].lower() == 'changetype':
+                line = line.lower().replace(': ntdsschemaadd',
+                                            ': add')
+                line = line.lower().replace(': ntdsschemamodify',
+                                            ': modify')
+
+            if values[0].lower() in ['rdnattid', 'subclassof',
+                                     'systemposssuperiors',
+                                     'systemmaycontain',
+                                     'systemauxiliaryclass']:
+                _, value = values
+
+                # The Microsoft updates contain some OIDs we don't recognize.
+                # Query the DB to see if we can work out the OID this update is
+                # referring to. If we find a match, then replace the OID with
+                # the ldapDisplayname
+                if '.' in value:
+                    res = samdb.search(base=samdb.get_schema_basedn(),
+                                       expression="(|(attributeId=%s)(governsId=%s))" %
+                                       (value, value),
+                                       attrs=['ldapDisplayName'])
+
+                    if len(res) != 1:
+                        ldif_op.unknown_oid = value
+                    else:
+                        display_name = res[0]['ldapDisplayName'][0]
+                        line = line.replace(value, ' ' + display_name)
+
+            # Microsoft has marked objects as defunct that Samba doesn't know about
+            if values[0].lower() == 'isdefunct' and values[1].strip().lower() == 'true':
+                ldif_op.is_defunct = True
+
+            # Samba has added the showInAdvancedViewOnly attribute to all objects,
+            # so rather than doing an add, we need to do a replace
+            if values[0].lower() == 'add' and values[1].strip().lower() == 'showinadvancedviewonly':
+                line = 'replace: showInAdvancedViewOnly'
+
+            # Add the line to the current LDIF operation (including the newline
+            # we stripped off at the start of the loop)
+            ldif_op.ldif += line + '\n'
+
+        return count
+
+
+    def _apply_update(self, samdb, update_file):
+        """Wrapper function for parsing an LDIF file and applying the updates"""
+
+        print("Applying %s updates..." % update_file)
+        path = setup_path('adprep')
+
+        ldif_file = None
+        try:
+            ldif_file = open(os.path.join(path, update_file))
+
+            count = self._apply_updates_in_file(samdb, ldif_file)
+
+        finally:
+            if ldif_file:
+                ldif_file.close()
+
+        print("%u changes applied" % count)
+
+        return count
+
+    def run(self, **kwargs):
+        from samba.schema import Schema
+
+        sambaopts = kwargs.get("sambaopts")
+        credopts = kwargs.get("credopts")
+        versionpts = kwargs.get("versionopts")
+        lp = sambaopts.get_loadparm()
+        creds = credopts.get_credentials(lp)
+        H = kwargs.get("H")
+        target_schema = kwargs.get("schema")
+
+        samdb = SamDB(url=H, session_info=system_session(), credentials=creds, lp=lp)
+
+        # work out the version of the target schema we're upgrading to
+        end = Schema.get_version(target_schema)
+
+        # work out the version of the schema we're currently using
+        res = samdb.search(base=samdb.get_schema_basedn(), scope=ldb.SCOPE_BASE,
+                           attrs=['objectVersion'])
+
+        if len(res) != 1:
+            raise CommandError('Could not determine current schema version')
+        start = int(res[0]['objectVersion'][0]) + 1
+
+        samdb.transaction_start()
+        count = 0
+
+        try:
+            # Apply the schema updates needed to move to the new schema version
+            for version in range(start, end + 1):
+                count += self._apply_update(samdb, 'Sch%d.ldf' % version)
+
+            if count > 0:
+                samdb.transaction_commit()
+                print("Schema successfully updated")
+            else:
+                print("No changes applied to schema")
+                samdb.transaction_cancel()
+        except Exception as e:
+            print("Exception: %s" % e)
+            print("Error encountered, aborting schema upgrade")
+            samdb.transaction_cancel()
+            raise CommandError('Failed to upgrade schema')
+
 class cmd_domain(SuperCommand):
     """Domain management."""
 
@@ -3869,3 +4095,4 @@ class cmd_domain(SuperCommand):
     subcommands["samba3upgrade"] = cmd_domain_samba3upgrade()
     subcommands["trust"] = cmd_domain_trust()
     subcommands["tombstones"] = cmd_domain_tombstones()
+    subcommands["schemaupgrade"] = cmd_domain_schema_upgrade()