1
0
mirror of https://github.com/ansible/awx.git synced 2024-10-31 15:21:13 +03:00

First pass at cleanup_deleted management command.

This commit is contained in:
Chris Church 2013-06-30 12:36:43 -04:00
parent 14ed59bedf
commit 6f0d644f5f
6 changed files with 318 additions and 10 deletions

View File

@ -1,23 +1,28 @@
# Copyright (c) 2013 AnsibleWorks, Inc.
# All Rights Reserved.
# Python
import json
# Django
from django.http import HttpResponse, Http404
from django.views.decorators.csrf import csrf_exempt
from awx.main.models import *
from django.contrib.auth.models import User
from awx.main.serializers import *
from awx.main.rbac import *
from awx.main.access import *
from django.utils.timezone import now
# Django REST Framework
from rest_framework.exceptions import PermissionDenied
from rest_framework import mixins
from rest_framework import generics
from rest_framework import permissions
from rest_framework.response import Response
from rest_framework import status
import exceptions
import datetime
import json as python_json
# AWX
from awx.main.models import *
from awx.main.serializers import *
from awx.main.rbac import *
from awx.main.access import *
# FIXME: machinery for auto-adding audit trail logs to all CREATE/EDITS
@ -128,7 +133,7 @@ class BaseSubList(BaseList):
if not organization.admins.filter(pk=request.user.pk).count() > 0:
raise PermissionDenied()
else:
raise exceptions.NotImplementedError()
raise NotImplementedError()
else:
if not check_user_access(request.user, type(obj), 'read', obj):
raise PermissionDenied()
@ -162,7 +167,7 @@ class BaseSubList(BaseList):
else:
# view didn't specify a way to get the pk from the URL, so not even trying
return Response(status=status.HTTP_400_BAD_REQUEST, data=python_json.dumps(dict(msg='object cannot be created')))
return Response(status=status.HTTP_400_BAD_REQUEST, data=json.dumps(dict(msg='object cannot be created')))
# we didn't have to create the object, so this is just associating the two objects together now...
# (or disassociating them)
@ -229,7 +234,7 @@ class BaseDetail(generics.RetrieveUpdateDestroyAPIView):
if isinstance(obj, PrimordialModel):
obj.mark_inactive()
elif type(obj) == User:
obj.username = "_deleted_%s_%s" % (str(datetime.time()), obj.username)
obj.username = "_deleted_%s_%s" % (now().isoformat(), obj.username)
obj.is_active = False
obj.save()
else:

View File

View File

View File

@ -0,0 +1,95 @@
# Copyright (c) 2013 AnsibleWorks, Inc.
# All Rights Reserved.
# Python
import datetime
import logging
from optparse import make_option
# Django
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from django.contrib.auth.models import User
from django.utils.dateparse import parse_datetime
from django.utils.timezone import now
# AWX
from awx.main.models import *
class Command(BaseCommand):
'''
Management command to cleanup deleted items.
'''
help = 'Cleanup deleted items from the database.'
args = '[<appname>, <appname.ModelName>, ...]'
option_list = BaseCommand.option_list + (
make_option('--days', dest='days', type='int', default=90, metavar='N',
help='Remove items deleted more than N days ago'),
make_option('--dry-run', dest='dry_run', action='store_true',
default=False, help='Dry run mode (show items that would '
'be removed)'),
)
def get_models(self, model):
if not model._meta.abstract:
yield model
for sub in model.__subclasses__():
for submodel in self.get_models(sub):
yield submodel
def cleanup_model(self, model):
name_field = None
active_field = None
for field in model._meta.fields:
if field.name in ('name', 'username'):
name_field = field.name
if field.name in ('is_active', 'active'):
active_field = field.name
if not name_field:
self.logger.warning('skipping model %s, no name field', model)
return
if not active_field:
self.logger.warning('skipping model %s, no active field', model)
return
qs = model.objects.filter(**{
active_field: False,
'%s__startswith' % name_field: '_deleted_',
})
self.logger.debug('cleaning up model %s', model)
for instance in qs:
dt = parse_datetime(getattr(instance, name_field).split('_')[2])
if not dt:
self.logger.warning('unable to find deleted timestamp in %s '
'field', name_field)
elif dt >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, instance)
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, instance)
if not self.dry_run:
instance.delete()
def init_logging(self):
log_levels = dict(enumerate([logging.ERROR, logging.INFO,
logging.DEBUG, 0]))
self.logger = logging.getLogger('awx.main.commands.cleanup_deleted')
self.logger.setLevel(log_levels.get(self.verbosity, 0))
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter('%(message)s'))
self.logger.addHandler(handler)
self.logger.propagate = False
@transaction.commit_on_success
def handle(self, *args, **options):
self.verbosity = int(options.get('verbosity', 1))
self.init_logging()
self.days = int(options.get('days', 90))
self.dry_run = bool(options.get('dry_run', False))
# FIXME: Handle args to select models.
self.cutoff = now() - datetime.timedelta(days=self.days)
self.cleanup_model(User)
for model in self.get_models(PrimordialModel):
self.cleanup_model(model)

View File

@ -5,6 +5,7 @@ from awx.main.tests.organizations import OrganizationsTest
from awx.main.tests.users import UsersTest
from awx.main.tests.inventory import InventoryTest
from awx.main.tests.projects import ProjectsTest
from awx.main.tests.commands import *
from awx.main.tests.scripts import *
from awx.main.tests.tasks import RunJobTest
from awx.main.tests.licenses import LicenseTests

207
awx/main/tests/commands.py Normal file
View File

@ -0,0 +1,207 @@
# Copyright (c) 2013 AnsibleWorks, Inc.
# All Rights Reserved.
# Python
import json
import os
import StringIO
import sys
import tempfile
# Django
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import CommandError
from django.utils.timezone import now
# AWX
from awx.main.models import *
from awx.main.tests.base import BaseTest
__all__ = ['CleanupDeletedTest']
class BaseCommandTest(BaseTest):
'''
Base class for tests that run management commands.
'''
def setUp(self):
super(BaseCommandTest, self).setUp()
self._sys_path = [x for x in sys.path]
self._environ = dict(os.environ.items())
self._temp_files = []
def tearDown(self):
super(BaseCommandTest, self).tearDown()
sys.path = self._sys_path
for k,v in self._environ.items():
if os.environ.get(k, None) != v:
os.environ[k] = v
for k,v in os.environ.items():
if k not in self._environ.keys():
del os.environ[k]
for tf in self._temp_files:
if os.path.exists(tf):
os.remove(tf)
def run_command(self, name, *args, **options):
'''
Run a management command and capture its stdout/stderr along with any
exceptions.
'''
command_runner = options.pop('command_runner', call_command)
stdin_fileobj = options.pop('stdin_fileobj', None)
options.setdefault('verbosity', 1)
options.setdefault('interactive', False)
original_stdin = sys.stdin
original_stdout = sys.stdout
original_stderr = sys.stderr
if stdin_fileobj:
sys.stdin = stdin_fileobj
sys.stdout = StringIO.StringIO()
sys.stderr = StringIO.StringIO()
result = None
try:
result = command_runner(name, *args, **options)
except Exception, e:
result = e
except SystemExit, e:
result = e
finally:
captured_stdout = sys.stdout.getvalue()
captured_stderr = sys.stderr.getvalue()
sys.stdin = original_stdin
sys.stdout = original_stdout
sys.stderr = original_stderr
return result, captured_stdout, captured_stderr
class CleanupDeletedTest(BaseCommandTest):
'''
Test cases for cleanup_deleted management command.
'''
def setUp(self):
super(CleanupDeletedTest, self).setUp()
self.setup_users()
self.organizations = self.make_organizations(self.super_django_user, 2)
self.projects = self.make_projects(self.normal_django_user, 2)
self.organizations[0].projects.add(self.projects[1])
self.organizations[1].projects.add(self.projects[0])
self.inventories = []
self.hosts = []
self.groups = []
for n, organization in enumerate(self.organizations):
inventory = Inventory.objects.create(name='inventory-%d' % n,
description='description for inventory %d' % n,
organization=organization,
variables=json.dumps({'n': n}) if n else '')
self.inventories.append(inventory)
hosts = []
for x in xrange(10):
if n > 0:
variables = json.dumps({'ho': 'hum-%d' % x})
else:
variables = ''
host = inventory.hosts.create(name='host-%02d-%02d.example.com' % (n, x),
inventory=inventory,
variables=variables)
hosts.append(host)
self.hosts.extend(hosts)
groups = []
for x in xrange(5):
if n > 0:
variables = json.dumps({'gee': 'whiz-%d' % x})
else:
variables = ''
group = inventory.groups.create(name='group-%d' % x,
inventory=inventory,
variables=variables)
groups.append(group)
group.hosts.add(hosts[x])
group.hosts.add(hosts[x + 5])
if n > 0 and x == 4:
group.parents.add(groups[3])
self.groups.extend(groups)
def get_model_counts(self):
def get_models(m):
if not m._meta.abstract:
yield m
for sub in m.__subclasses__():
for subm in get_models(sub):
yield subm
counts = {}
for model in get_models(PrimordialModel):
active = model.objects.filter(active=True).count()
inactive = model.objects.filter(active=False).count()
counts[model] = (active, inactive)
return counts
def test_cleanup_our_models(self):
# Test with nothing to be deleted.
counts_before = self.get_model_counts()
self.assertFalse(sum(x[1] for x in counts_before.values()))
result, stdout, stderr = self.run_command('cleanup_deleted')
self.assertEqual(result, None)
counts_after = self.get_model_counts()
self.assertEqual(counts_before, counts_after)
# "Delete" some hosts.
for host in Host.objects.all():
host.mark_inactive()
# With no parameters, "days" defaults to 90, which won't cleanup any of
# the hosts we just removed.
counts_before = self.get_model_counts()
self.assertTrue(sum(x[1] for x in counts_before.values()))
result, stdout, stderr = self.run_command('cleanup_deleted')
self.assertEqual(result, None)
counts_after = self.get_model_counts()
self.assertEqual(counts_before, counts_after)
# Even with days=1, the hosts will remain.
counts_before = self.get_model_counts()
self.assertTrue(sum(x[1] for x in counts_before.values()))
result, stdout, stderr = self.run_command('cleanup_deleted', days=1)
self.assertEqual(result, None)
counts_after = self.get_model_counts()
self.assertEqual(counts_before, counts_after)
# With days=0, the hosts will be deleted.
counts_before = self.get_model_counts()
self.assertTrue(sum(x[1] for x in counts_before.values()))
result, stdout, stderr = self.run_command('cleanup_deleted', days=0)
self.assertEqual(result, None)
counts_after = self.get_model_counts()
self.assertNotEqual(counts_before, counts_after)
self.assertFalse(sum(x[1] for x in counts_after.values()))
def get_user_counts(self):
active = User.objects.filter(is_active=True).count()
inactive = User.objects.filter(is_active=False).count()
return active, inactive
def test_cleanup_user_model(self):
# Test with nothing to be deleted.
counts_before = self.get_user_counts()
self.assertFalse(counts_before[1])
result, stdout, stderr = self.run_command('cleanup_deleted')
self.assertEqual(result, None)
counts_after = self.get_user_counts()
self.assertEqual(counts_before, counts_after)
# "Delete some users".
for user in User.objects.all():
user.username = "_deleted_%s_%s" % (now().isoformat(), user.username)
user.is_active = False
user.save()
# With days=1, no users will be deleted.
counts_before = self.get_user_counts()
self.assertTrue(counts_before[1])
result, stdout, stderr = self.run_command('cleanup_deleted', days=1)
self.assertEqual(result, None)
counts_after = self.get_user_counts()
self.assertEqual(counts_before, counts_after)
# With days=0, inactive users will be deleted.
counts_before = self.get_user_counts()
self.assertTrue(counts_before[1])
result, stdout, stderr = self.run_command('cleanup_deleted', days=0)
self.assertEqual(result, None)
counts_after = self.get_user_counts()
self.assertNotEqual(counts_before, counts_after)
self.assertFalse(counts_after[1])