From 6f0d644f5f40b42c0af89ed20255b2594ef172e0 Mon Sep 17 00:00:00 2001 From: Chris Church Date: Sun, 30 Jun 2013 12:36:43 -0400 Subject: [PATCH] First pass at cleanup_deleted management command. --- awx/main/base_views.py | 25 ++- awx/main/management/__init__.py | 0 awx/main/management/commands/__init__.py | 0 .../management/commands/cleanup_deleted.py | 95 ++++++++ awx/main/tests/__init__.py | 1 + awx/main/tests/commands.py | 207 ++++++++++++++++++ 6 files changed, 318 insertions(+), 10 deletions(-) create mode 100644 awx/main/management/__init__.py create mode 100644 awx/main/management/commands/__init__.py create mode 100644 awx/main/management/commands/cleanup_deleted.py create mode 100644 awx/main/tests/commands.py diff --git a/awx/main/base_views.py b/awx/main/base_views.py index 7fc5a467e5..e0910a6da9 100644 --- a/awx/main/base_views.py +++ b/awx/main/base_views.py @@ -1,23 +1,28 @@ # Copyright (c) 2013 AnsibleWorks, Inc. # All Rights Reserved. +# Python +import json +# Django from django.http import HttpResponse, Http404 from django.views.decorators.csrf import csrf_exempt -from awx.main.models import * from django.contrib.auth.models import User -from awx.main.serializers import * -from awx.main.rbac import * -from awx.main.access import * +from django.utils.timezone import now + +# Django REST Framework from rest_framework.exceptions import PermissionDenied from rest_framework import mixins from rest_framework import generics from rest_framework import permissions from rest_framework.response import Response from rest_framework import status -import exceptions -import datetime -import json as python_json + +# AWX +from awx.main.models import * +from awx.main.serializers import * +from awx.main.rbac import * +from awx.main.access import * # FIXME: machinery for auto-adding audit trail logs to all CREATE/EDITS @@ -128,7 +133,7 @@ class BaseSubList(BaseList): if not organization.admins.filter(pk=request.user.pk).count() > 0: raise PermissionDenied() else: - raise exceptions.NotImplementedError() + raise NotImplementedError() else: if not check_user_access(request.user, type(obj), 'read', obj): raise PermissionDenied() @@ -162,7 +167,7 @@ class BaseSubList(BaseList): else: # view didn't specify a way to get the pk from the URL, so not even trying - return Response(status=status.HTTP_400_BAD_REQUEST, data=python_json.dumps(dict(msg='object cannot be created'))) + return Response(status=status.HTTP_400_BAD_REQUEST, data=json.dumps(dict(msg='object cannot be created'))) # we didn't have to create the object, so this is just associating the two objects together now... # (or disassociating them) @@ -229,7 +234,7 @@ class BaseDetail(generics.RetrieveUpdateDestroyAPIView): if isinstance(obj, PrimordialModel): obj.mark_inactive() elif type(obj) == User: - obj.username = "_deleted_%s_%s" % (str(datetime.time()), obj.username) + obj.username = "_deleted_%s_%s" % (now().isoformat(), obj.username) obj.is_active = False obj.save() else: diff --git a/awx/main/management/__init__.py b/awx/main/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/awx/main/management/commands/__init__.py b/awx/main/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/awx/main/management/commands/cleanup_deleted.py b/awx/main/management/commands/cleanup_deleted.py new file mode 100644 index 0000000000..84ead9b585 --- /dev/null +++ b/awx/main/management/commands/cleanup_deleted.py @@ -0,0 +1,95 @@ +# Copyright (c) 2013 AnsibleWorks, Inc. +# All Rights Reserved. + +# Python +import datetime +import logging +from optparse import make_option + +# Django +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction +from django.contrib.auth.models import User +from django.utils.dateparse import parse_datetime +from django.utils.timezone import now + +# AWX +from awx.main.models import * + +class Command(BaseCommand): + ''' + Management command to cleanup deleted items. + ''' + + help = 'Cleanup deleted items from the database.' + args = '[, , ...]' + + option_list = BaseCommand.option_list + ( + make_option('--days', dest='days', type='int', default=90, metavar='N', + help='Remove items deleted more than N days ago'), + make_option('--dry-run', dest='dry_run', action='store_true', + default=False, help='Dry run mode (show items that would ' + 'be removed)'), + ) + + def get_models(self, model): + if not model._meta.abstract: + yield model + for sub in model.__subclasses__(): + for submodel in self.get_models(sub): + yield submodel + + def cleanup_model(self, model): + name_field = None + active_field = None + for field in model._meta.fields: + if field.name in ('name', 'username'): + name_field = field.name + if field.name in ('is_active', 'active'): + active_field = field.name + if not name_field: + self.logger.warning('skipping model %s, no name field', model) + return + if not active_field: + self.logger.warning('skipping model %s, no active field', model) + return + qs = model.objects.filter(**{ + active_field: False, + '%s__startswith' % name_field: '_deleted_', + }) + self.logger.debug('cleaning up model %s', model) + for instance in qs: + dt = parse_datetime(getattr(instance, name_field).split('_')[2]) + if not dt: + self.logger.warning('unable to find deleted timestamp in %s ' + 'field', name_field) + elif dt >= self.cutoff: + action_text = 'would skip' if self.dry_run else 'skipping' + self.logger.debug('%s %s', action_text, instance) + else: + action_text = 'would delete' if self.dry_run else 'deleting' + self.logger.info('%s %s', action_text, instance) + if not self.dry_run: + instance.delete() + + def init_logging(self): + log_levels = dict(enumerate([logging.ERROR, logging.INFO, + logging.DEBUG, 0])) + self.logger = logging.getLogger('awx.main.commands.cleanup_deleted') + self.logger.setLevel(log_levels.get(self.verbosity, 0)) + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(message)s')) + self.logger.addHandler(handler) + self.logger.propagate = False + + @transaction.commit_on_success + def handle(self, *args, **options): + self.verbosity = int(options.get('verbosity', 1)) + self.init_logging() + self.days = int(options.get('days', 90)) + self.dry_run = bool(options.get('dry_run', False)) + # FIXME: Handle args to select models. + self.cutoff = now() - datetime.timedelta(days=self.days) + self.cleanup_model(User) + for model in self.get_models(PrimordialModel): + self.cleanup_model(model) diff --git a/awx/main/tests/__init__.py b/awx/main/tests/__init__.py index a2b8640f0a..9da6477668 100644 --- a/awx/main/tests/__init__.py +++ b/awx/main/tests/__init__.py @@ -5,6 +5,7 @@ from awx.main.tests.organizations import OrganizationsTest from awx.main.tests.users import UsersTest from awx.main.tests.inventory import InventoryTest from awx.main.tests.projects import ProjectsTest +from awx.main.tests.commands import * from awx.main.tests.scripts import * from awx.main.tests.tasks import RunJobTest from awx.main.tests.licenses import LicenseTests diff --git a/awx/main/tests/commands.py b/awx/main/tests/commands.py new file mode 100644 index 0000000000..d3c0d2bfcc --- /dev/null +++ b/awx/main/tests/commands.py @@ -0,0 +1,207 @@ +# Copyright (c) 2013 AnsibleWorks, Inc. +# All Rights Reserved. + +# Python +import json +import os +import StringIO +import sys +import tempfile + +# Django +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import CommandError +from django.utils.timezone import now + +# AWX +from awx.main.models import * +from awx.main.tests.base import BaseTest + +__all__ = ['CleanupDeletedTest'] + +class BaseCommandTest(BaseTest): + ''' + Base class for tests that run management commands. + ''' + + def setUp(self): + super(BaseCommandTest, self).setUp() + self._sys_path = [x for x in sys.path] + self._environ = dict(os.environ.items()) + self._temp_files = [] + + def tearDown(self): + super(BaseCommandTest, self).tearDown() + sys.path = self._sys_path + for k,v in self._environ.items(): + if os.environ.get(k, None) != v: + os.environ[k] = v + for k,v in os.environ.items(): + if k not in self._environ.keys(): + del os.environ[k] + for tf in self._temp_files: + if os.path.exists(tf): + os.remove(tf) + + def run_command(self, name, *args, **options): + ''' + Run a management command and capture its stdout/stderr along with any + exceptions. + ''' + command_runner = options.pop('command_runner', call_command) + stdin_fileobj = options.pop('stdin_fileobj', None) + options.setdefault('verbosity', 1) + options.setdefault('interactive', False) + original_stdin = sys.stdin + original_stdout = sys.stdout + original_stderr = sys.stderr + if stdin_fileobj: + sys.stdin = stdin_fileobj + sys.stdout = StringIO.StringIO() + sys.stderr = StringIO.StringIO() + result = None + try: + result = command_runner(name, *args, **options) + except Exception, e: + result = e + except SystemExit, e: + result = e + finally: + captured_stdout = sys.stdout.getvalue() + captured_stderr = sys.stderr.getvalue() + sys.stdin = original_stdin + sys.stdout = original_stdout + sys.stderr = original_stderr + return result, captured_stdout, captured_stderr + +class CleanupDeletedTest(BaseCommandTest): + ''' + Test cases for cleanup_deleted management command. + ''' + + def setUp(self): + super(CleanupDeletedTest, self).setUp() + self.setup_users() + self.organizations = self.make_organizations(self.super_django_user, 2) + self.projects = self.make_projects(self.normal_django_user, 2) + self.organizations[0].projects.add(self.projects[1]) + self.organizations[1].projects.add(self.projects[0]) + self.inventories = [] + self.hosts = [] + self.groups = [] + for n, organization in enumerate(self.organizations): + inventory = Inventory.objects.create(name='inventory-%d' % n, + description='description for inventory %d' % n, + organization=organization, + variables=json.dumps({'n': n}) if n else '') + self.inventories.append(inventory) + hosts = [] + for x in xrange(10): + if n > 0: + variables = json.dumps({'ho': 'hum-%d' % x}) + else: + variables = '' + host = inventory.hosts.create(name='host-%02d-%02d.example.com' % (n, x), + inventory=inventory, + variables=variables) + hosts.append(host) + self.hosts.extend(hosts) + groups = [] + for x in xrange(5): + if n > 0: + variables = json.dumps({'gee': 'whiz-%d' % x}) + else: + variables = '' + group = inventory.groups.create(name='group-%d' % x, + inventory=inventory, + variables=variables) + groups.append(group) + group.hosts.add(hosts[x]) + group.hosts.add(hosts[x + 5]) + if n > 0 and x == 4: + group.parents.add(groups[3]) + self.groups.extend(groups) + + def get_model_counts(self): + def get_models(m): + if not m._meta.abstract: + yield m + for sub in m.__subclasses__(): + for subm in get_models(sub): + yield subm + counts = {} + for model in get_models(PrimordialModel): + active = model.objects.filter(active=True).count() + inactive = model.objects.filter(active=False).count() + counts[model] = (active, inactive) + return counts + + def test_cleanup_our_models(self): + # Test with nothing to be deleted. + counts_before = self.get_model_counts() + self.assertFalse(sum(x[1] for x in counts_before.values())) + result, stdout, stderr = self.run_command('cleanup_deleted') + self.assertEqual(result, None) + counts_after = self.get_model_counts() + self.assertEqual(counts_before, counts_after) + # "Delete" some hosts. + for host in Host.objects.all(): + host.mark_inactive() + # With no parameters, "days" defaults to 90, which won't cleanup any of + # the hosts we just removed. + counts_before = self.get_model_counts() + self.assertTrue(sum(x[1] for x in counts_before.values())) + result, stdout, stderr = self.run_command('cleanup_deleted') + self.assertEqual(result, None) + counts_after = self.get_model_counts() + self.assertEqual(counts_before, counts_after) + # Even with days=1, the hosts will remain. + counts_before = self.get_model_counts() + self.assertTrue(sum(x[1] for x in counts_before.values())) + result, stdout, stderr = self.run_command('cleanup_deleted', days=1) + self.assertEqual(result, None) + counts_after = self.get_model_counts() + self.assertEqual(counts_before, counts_after) + # With days=0, the hosts will be deleted. + counts_before = self.get_model_counts() + self.assertTrue(sum(x[1] for x in counts_before.values())) + result, stdout, stderr = self.run_command('cleanup_deleted', days=0) + self.assertEqual(result, None) + counts_after = self.get_model_counts() + self.assertNotEqual(counts_before, counts_after) + self.assertFalse(sum(x[1] for x in counts_after.values())) + + def get_user_counts(self): + active = User.objects.filter(is_active=True).count() + inactive = User.objects.filter(is_active=False).count() + return active, inactive + + def test_cleanup_user_model(self): + # Test with nothing to be deleted. + counts_before = self.get_user_counts() + self.assertFalse(counts_before[1]) + result, stdout, stderr = self.run_command('cleanup_deleted') + self.assertEqual(result, None) + counts_after = self.get_user_counts() + self.assertEqual(counts_before, counts_after) + # "Delete some users". + for user in User.objects.all(): + user.username = "_deleted_%s_%s" % (now().isoformat(), user.username) + user.is_active = False + user.save() + # With days=1, no users will be deleted. + counts_before = self.get_user_counts() + self.assertTrue(counts_before[1]) + result, stdout, stderr = self.run_command('cleanup_deleted', days=1) + self.assertEqual(result, None) + counts_after = self.get_user_counts() + self.assertEqual(counts_before, counts_after) + # With days=0, inactive users will be deleted. + counts_before = self.get_user_counts() + self.assertTrue(counts_before[1]) + result, stdout, stderr = self.run_command('cleanup_deleted', days=0) + self.assertEqual(result, None) + counts_after = self.get_user_counts() + self.assertNotEqual(counts_before, counts_after) + self.assertFalse(counts_after[1])