Do cleanup_deleted on migrate. Re-ordered active flag removal to be before system job template creation.

Also removed active flag deletes from remaining cleanup_deleted
management command as they will no longer be needed - but the
deletes of the authentication tokens as well as potentially disabled
users are still necessary, so the cleanup_deleted command will continue
to exist.

Reordering of the active flag removal to happen before the system job
template creation is necessary since the system job template creation
hits the license checker which at some point runs queries that depend on
the active flag, and with that code changing to not use the active flag,
we need to do the removal before we run this code.

awx/main/management/commands/cleanup_deleted.py

@@ -111,8 +111,6 @@ class Command(BaseCommand):
 
         n_deleted_items = 0
         n_deleted_items += self.cleanup_model(User)
-        for model in self.get_models(PrimordialModel):
-            n_deleted_items += self.cleanup_model(model)
 
         if not self.dry_run:
             self.logger.log(99, "Removed %d items", n_deleted_items)

awx/main/migrations/0005_v300_active_flag_removal.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from awx.main.migrations import _cleanup_deleted as cleanup_deleted
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0004_v300_changes'),
+    ]
+
+    operations = [
+        migrations.RunPython(cleanup_deleted.cleanup_deleted),
+
+        migrations.RemoveField(
+            model_name='credential',
+            name='active',
+        ),
+        migrations.RemoveField(
+            model_name='custominventoryscript',
+            name='active',
+        ),
+        migrations.RemoveField(
+            model_name='group',
+            name='active',
+        ),
+        migrations.RemoveField(
+            model_name='host',
+            name='active',
+        ),
+        migrations.RemoveField(
+            model_name='inventory',
+            name='active',
+        ),
+        migrations.RemoveField(
+            model_name='notifier',
+            name='active',
+        ),
+        migrations.RemoveField(
+            model_name='organization',
+            name='active',
+        ),
+        migrations.RemoveField(
+            model_name='permission',
+            name='active',
+        ),
+        migrations.RemoveField(
+            model_name='schedule',
+            name='active',
+        ),
+        migrations.RemoveField(
+            model_name='team',
+            name='active',
+        ),
+        migrations.RemoveField(
+            model_name='unifiedjob',
+            name='active',
+        ),
+        migrations.RemoveField(
+            model_name='unifiedjobtemplate',
+            name='active',
+        ),
+    ]

awx/main/migrations/0006_v300_active_flag_removal.py

@@ -1,16 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from awx.main.migrations import _rbac as rbac
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('main', '0005_v300_changes'),
-    ]
-
-    operations = [
-        # This is a placeholder for our future active flag removal work
-    ]

awx/main/migrations/0006_v300_changes.py

@@ -107,7 +107,7 @@ def create_system_job_templates(apps, schema_editor):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('main', '0004_v300_changes'),
+        ('main', '0005_v300_active_flag_removal'),
     ]
 
     operations = [

awx/main/migrations/0007_v300_rbac_changes.py

@@ -14,7 +14,7 @@ class Migration(migrations.Migration):
         ('taggit', '0002_auto_20150616_2121'),
         ('contenttypes', '0002_remove_content_type_name'),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-        ('main', '0006_v300_active_flag_removal'),
+        ('main', '0006_v300_changes'),
     ]
 
     operations = [

awx/main/migrations/_cleanup_deleted.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2016 Ansible, Inc.
+# All Rights Reserved.
+
+# Python
+import logging
+
+# Django
+from django.db import transaction
+from django.utils.dateparse import parse_datetime
+
+def cleanup_deleted(apps, schema_editor):
+    logger = logging.getLogger('awx.main.migrations.cleanup_deleted')
+
+    def cleanup_model(model):
+
+        '''
+        Presume the '_deleted_' string to be in the 'name' field unless considering the User model.
+        When considering the User model, presume the '_d_' string to be in the 'username' field.
+        '''
+        logger.debug('cleaning up model %s', model)
+
+        name_field = 'name'
+        name_prefix = '_deleted_'
+        active_field = None
+        n_deleted_items = 0
+        for field in model._meta.fields:
+            if field.name in ('is_active', 'active'):
+                active_field = field.name
+            if field.name == 'is_active': # is User model
+                name_field = 'username'
+                name_prefix = '_d_'
+        if not active_field:
+            logger.warning('skipping model %s, no active field', model)
+            return n_deleted_items
+        qs = model.objects.filter(**{
+            active_field: False,
+            '%s__startswith' % name_field: name_prefix,
+        })
+        pks_to_delete = set()
+        for instance in qs.iterator():
+            dt = parse_datetime(getattr(instance, name_field).split('_')[2])
+            if not dt:
+                logger.warning('unable to find deleted timestamp in %s field', name_field)
+            else:
+                action_text = 'deleting'
+                logger.info('%s %s', action_text, instance)
+                n_deleted_items += 1
+                instance.delete()
+
+            # Cleanup objects in batches instead of deleting each one individually.
+            if len(pks_to_delete) >= 50:
+                model.objects.filter(pk__in=pks_to_delete).delete()
+                pks_to_delete.clear()
+        if len(pks_to_delete):
+            model.objects.filter(pk__in=pks_to_delete).delete()
+        return n_deleted_items
+
+    logger = logging.getLogger('awx.main.commands.cleanup_deleted')
+    handler = logging.StreamHandler()
+    handler.setFormatter(logging.Formatter('%(message)s'))
+    logger.addHandler(handler)
+    logger.propagate = False
+
+    with transaction.atomic():
+        n_deleted_items = 0
+
+        models = [
+            apps.get_model('auth', "User"),
+            apps.get_model('main', 'Credential'),
+            apps.get_model('main', 'CustomInventoryScript'),
+            apps.get_model('main', 'Group'),
+            apps.get_model('main', 'Host'),
+            apps.get_model('main', 'Inventory'),
+            apps.get_model('main', 'Notifier'),
+            apps.get_model('main', 'Organization'),
+            apps.get_model('main', 'Permission'),
+            apps.get_model('main', 'Schedule'),
+            apps.get_model('main', 'Team'),
+            apps.get_model('main', 'UnifiedJob'),
+            apps.get_model('main', 'UnifiedJobTemplate'),
+        ]
+
+        for model in models:
+            n_deleted_items += cleanup_model(model)
+            logger.log(99, "Removed %d items", n_deleted_items)