From 4825b2a6fcb8fb325dbbc8190eb5195637b3e677 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Wed, 9 Mar 2016 15:38:57 -0500 Subject: [PATCH] Do cleanup_deleted on migrate. Re-ordered active flag removal to be before system job template creation. Also removed active flag deletes from remaining cleanup_deleted management command as they will no longer be needed - but the deletes of the authentication tokens as well as potentially disabled users are still necessary, so the cleanup_deleted command will continue to exist. Reordering of the active flag removal to happen before the system job template creation is necessary since the system job template creation hits the license checker which at some point runs queries that depend on the active flag, and with that code changing to not use the active flag, we need to do the removal before we run this code. --- .../management/commands/cleanup_deleted.py | 2 - .../0005_v300_active_flag_removal.py | 65 ++++++++++++++ .../0006_v300_active_flag_removal.py | 16 ---- ...5_v300_changes.py => 0006_v300_changes.py} | 2 +- awx/main/migrations/0007_v300_rbac_changes.py | 2 +- awx/main/migrations/_cleanup_deleted.py | 85 +++++++++++++++++++ 6 files changed, 152 insertions(+), 20 deletions(-) create mode 100644 awx/main/migrations/0005_v300_active_flag_removal.py delete mode 100644 awx/main/migrations/0006_v300_active_flag_removal.py rename awx/main/migrations/{0005_v300_changes.py => 0006_v300_changes.py} (98%) create mode 100644 awx/main/migrations/_cleanup_deleted.py diff --git a/awx/main/management/commands/cleanup_deleted.py b/awx/main/management/commands/cleanup_deleted.py index b6fd5360e5..7fd726be7f 100644 --- a/awx/main/management/commands/cleanup_deleted.py +++ b/awx/main/management/commands/cleanup_deleted.py @@ -111,8 +111,6 @@ class Command(BaseCommand): n_deleted_items = 0 n_deleted_items += self.cleanup_model(User) - for model in self.get_models(PrimordialModel): - n_deleted_items += self.cleanup_model(model) if not self.dry_run: self.logger.log(99, "Removed %d items", n_deleted_items) diff --git a/awx/main/migrations/0005_v300_active_flag_removal.py b/awx/main/migrations/0005_v300_active_flag_removal.py new file mode 100644 index 0000000000..adc6a0cddf --- /dev/null +++ b/awx/main/migrations/0005_v300_active_flag_removal.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from awx.main.migrations import _cleanup_deleted as cleanup_deleted +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0004_v300_changes'), + ] + + operations = [ + migrations.RunPython(cleanup_deleted.cleanup_deleted), + + migrations.RemoveField( + model_name='credential', + name='active', + ), + migrations.RemoveField( + model_name='custominventoryscript', + name='active', + ), + migrations.RemoveField( + model_name='group', + name='active', + ), + migrations.RemoveField( + model_name='host', + name='active', + ), + migrations.RemoveField( + model_name='inventory', + name='active', + ), + migrations.RemoveField( + model_name='notifier', + name='active', + ), + migrations.RemoveField( + model_name='organization', + name='active', + ), + migrations.RemoveField( + model_name='permission', + name='active', + ), + migrations.RemoveField( + model_name='schedule', + name='active', + ), + migrations.RemoveField( + model_name='team', + name='active', + ), + migrations.RemoveField( + model_name='unifiedjob', + name='active', + ), + migrations.RemoveField( + model_name='unifiedjobtemplate', + name='active', + ), + ] diff --git a/awx/main/migrations/0006_v300_active_flag_removal.py b/awx/main/migrations/0006_v300_active_flag_removal.py deleted file mode 100644 index 795db642b2..0000000000 --- a/awx/main/migrations/0006_v300_active_flag_removal.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from awx.main.migrations import _rbac as rbac -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0005_v300_changes'), - ] - - operations = [ - # This is a placeholder for our future active flag removal work - ] diff --git a/awx/main/migrations/0005_v300_changes.py b/awx/main/migrations/0006_v300_changes.py similarity index 98% rename from awx/main/migrations/0005_v300_changes.py rename to awx/main/migrations/0006_v300_changes.py index e350c881f2..4162ff3bd1 100644 --- a/awx/main/migrations/0005_v300_changes.py +++ b/awx/main/migrations/0006_v300_changes.py @@ -107,7 +107,7 @@ def create_system_job_templates(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('main', '0004_v300_changes'), + ('main', '0005_v300_active_flag_removal'), ] operations = [ diff --git a/awx/main/migrations/0007_v300_rbac_changes.py b/awx/main/migrations/0007_v300_rbac_changes.py index f77d4c026a..b1aae399d3 100644 --- a/awx/main/migrations/0007_v300_rbac_changes.py +++ b/awx/main/migrations/0007_v300_rbac_changes.py @@ -14,7 +14,7 @@ class Migration(migrations.Migration): ('taggit', '0002_auto_20150616_2121'), ('contenttypes', '0002_remove_content_type_name'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('main', '0006_v300_active_flag_removal'), + ('main', '0006_v300_changes'), ] operations = [ diff --git a/awx/main/migrations/_cleanup_deleted.py b/awx/main/migrations/_cleanup_deleted.py new file mode 100644 index 0000000000..db187efeb4 --- /dev/null +++ b/awx/main/migrations/_cleanup_deleted.py @@ -0,0 +1,85 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +# Python +import logging + +# Django +from django.db import transaction +from django.utils.dateparse import parse_datetime + +def cleanup_deleted(apps, schema_editor): + logger = logging.getLogger('awx.main.migrations.cleanup_deleted') + + def cleanup_model(model): + + ''' + Presume the '_deleted_' string to be in the 'name' field unless considering the User model. + When considering the User model, presume the '_d_' string to be in the 'username' field. + ''' + logger.debug('cleaning up model %s', model) + + name_field = 'name' + name_prefix = '_deleted_' + active_field = None + n_deleted_items = 0 + for field in model._meta.fields: + if field.name in ('is_active', 'active'): + active_field = field.name + if field.name == 'is_active': # is User model + name_field = 'username' + name_prefix = '_d_' + if not active_field: + logger.warning('skipping model %s, no active field', model) + return n_deleted_items + qs = model.objects.filter(**{ + active_field: False, + '%s__startswith' % name_field: name_prefix, + }) + pks_to_delete = set() + for instance in qs.iterator(): + dt = parse_datetime(getattr(instance, name_field).split('_')[2]) + if not dt: + logger.warning('unable to find deleted timestamp in %s field', name_field) + else: + action_text = 'deleting' + logger.info('%s %s', action_text, instance) + n_deleted_items += 1 + instance.delete() + + # Cleanup objects in batches instead of deleting each one individually. + if len(pks_to_delete) >= 50: + model.objects.filter(pk__in=pks_to_delete).delete() + pks_to_delete.clear() + if len(pks_to_delete): + model.objects.filter(pk__in=pks_to_delete).delete() + return n_deleted_items + + logger = logging.getLogger('awx.main.commands.cleanup_deleted') + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(message)s')) + logger.addHandler(handler) + logger.propagate = False + + with transaction.atomic(): + n_deleted_items = 0 + + models = [ + apps.get_model('auth', "User"), + apps.get_model('main', 'Credential'), + apps.get_model('main', 'CustomInventoryScript'), + apps.get_model('main', 'Group'), + apps.get_model('main', 'Host'), + apps.get_model('main', 'Inventory'), + apps.get_model('main', 'Notifier'), + apps.get_model('main', 'Organization'), + apps.get_model('main', 'Permission'), + apps.get_model('main', 'Schedule'), + apps.get_model('main', 'Team'), + apps.get_model('main', 'UnifiedJob'), + apps.get_model('main', 'UnifiedJobTemplate'), + ] + + for model in models: + n_deleted_items += cleanup_model(model) + logger.log(99, "Removed %d items", n_deleted_items)