From 59423df95da8a0cd33b8073dbb6d3ecd0221b83c Mon Sep 17 00:00:00 2001 From: Chris Church Date: Sun, 8 Sep 2013 22:15:03 -0400 Subject: [PATCH] AC-132. Implement scm_update_on_launch and prevent simultaneous updates of associated projects and jobs. --- awx/main/models/__init__.py | 12 ++-- awx/main/tasks.py | 104 +++++++++++++++++++++++++------ awx/main/tests/projects.py | 119 ++++++++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 24 deletions(-) diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 5f068ae4f5..64729ee249 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -603,7 +603,7 @@ class Project(CommonModel): ) scm_delete_on_next_update = models.BooleanField( default=False, - editable=True, + editable=False, ) scm_update_on_launch = models.BooleanField( default=False, @@ -660,8 +660,6 @@ class Project(CommonModel): ) # FIXME: Still need to implement: - # - scm_update_on_launch - # - prevent simultaneous updates of project and running jobs using project # - masking passwords in project update args/stdout def save(self, *args, **kwargs): @@ -877,6 +875,9 @@ class ProjectUpdate(PrimordialModel): editable=False, ) + def __unicode__(self): + return u'%s-%s-%s' % (self.name, self.id, self.status) + def save(self, *args, **kwargs): # Get status before save... status_before = self.status or 'new' @@ -922,8 +923,6 @@ class ProjectUpdate(PrimordialModel): def start(self, **kwargs): from awx.main.tasks import RunProjectUpdate - if not self.can_start: - return False needed = self.project.scm_passwords_needed opts = dict([(field, kwargs.get(field, '')) for field in needed]) if not all(opts.values()): @@ -1233,6 +1232,9 @@ class Job(CommonModelNameNotUnique): def get_absolute_url(self): return reverse('main:job_detail', args=(self.pk,)) + def __unicode__(self): + return u'%s-%s-%s' % (self.name, self.id, self.status) + def save(self, *args, **kwargs): self.failed = bool(self.status in ('failed', 'error', 'canceled')) super(Job, self).save(*args, **kwargs) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index ee34c95604..f33fa92fc7 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -9,6 +9,7 @@ import logging import os import re import subprocess +import stat import tempfile import time import traceback @@ -74,11 +75,11 @@ class BaseTask(Task): if hasattr(project, 'scm_key_data'): ssh_key_data = decrypt_field(project, 'scm_key_data') if ssh_key_data: - # FIXME: File permissions? handle, path = tempfile.mkstemp() f = os.fdopen(handle, 'w') f.write(ssh_key_data) f.close() + os.chmod(stat.S_IRUSR|stat.S_IWUSR) return path else: return '' @@ -331,10 +332,58 @@ class RunJob(BaseTask): ''' Hook for checking job before running. ''' - if not super(RunJob, self).pre_run_check(job, **kwargs): - return False - # FIXME: Check if job is waiting on any projects that are being updated. - return True + project_update = None + while True: + pk = job.pk + if job.status in ('pending', 'waiting'): + project = job.project + pu_qs = project.project_updates.filter(status__in=('pending', 'running')) + # Refresh the current project_update instance (if set). + if project_update: + try: + project_update = project.project_updates.filter(pk=project_update.pk)[0] + except IndexError: + msg = 'Unable to check project update.' + job = self.update_model(pk, status='error', + result_traceback=msg) + return False + + # If the job needs to update the project first (and there is no + # specific project update defined). + if not project_update and project.scm_update_on_launch: + job = self.update_model(pk, status='waiting') + try: + project_update = pu_qs[0] + except IndexError: + kw = dict(kwargs.items()) + project_update = project.update(**kw) + if not project_update: + msg = 'Unable to update project before launch.' + job = self.update_model(pk, status='error', + result_traceback=msg) + return False + #print 'job %d waiting on project update %d' % (pk, project_update.pk) + time.sleep(2.0) + # If project update has failed, abort the job. + elif project_update and project_update.failed: + msg = 'Project update failed with status = %s.' % project_update.status + job = self.update_model(pk, status='error', + result_traceback=msg) + return False + # Check if blocked by any other active project updates. + elif pu_qs.count(): + #print 'job %d waiting on' % pk, pu_qs + job = self.update_model(pk, status='waiting') + time.sleep(4.0) + # Otherwise continue running the job. + else: + job = self.update_model(pk, status='pending') + return True + elif job.cancel_flag: + job = self.update_model(pk, status='canceled') + return False + else: + return False def post_run_hook(self, job): ''' @@ -356,11 +405,12 @@ class RunProjectUpdate(BaseTask): passwords = super(RunProjectUpdate, self).build_passwords(project_update, **kwargs) project = project_update.project - value = decrypt_field(project, 'scm_key_unlock') + value = kwargs.get('scm_key_unlock', decrypt_field(project, 'scm_key_unlock')) if value not in ('', 'ASK'): passwords['ssh_key_unlock'] = value passwords['scm_username'] = project.scm_username - passwords['scm_password'] = decrypt_field(project, 'scm_password') + passwords['scm_password'] = kwargs.get('scm_password', + decrypt_field(project, 'scm_password')) return passwords def build_env(self, project_update, **kwargs): @@ -391,19 +441,19 @@ class RunProjectUpdate(BaseTask): optionally using ssh-agent for public/private key authentication. ''' args = ['ansible-playbook', '-i', 'localhost,'] - # Since we specify -vvv and tasks use async polling, we should get some - # output regularly... args.append('-%s' % ('v' * 3)) extra_vars = {} project = project_update.project scm_url = project.scm_url if project.scm_username and project.scm_password not in ('ASK', ''): + scm_password = kwargs.get('scm_password', + decrypt_field(project, 'scm_password')) if project.scm_type == 'svn': extra_vars['scm_username'] = project.scm_username - extra_vars['scm_password'] = decrypt_field(project, 'scm_password') + extra_vars['scm_password'] = scm_password else: scm_url = self.update_url_auth(scm_url, project.scm_username, - decrypt_field(project, 'scm_password')) + scm_password) elif project.scm_username: if project.scm_type == 'svn': extra_vars['scm_username'] = project.scm_username @@ -436,6 +486,8 @@ class RunProjectUpdate(BaseTask): def get_password_prompts(self): d = super(RunProjectUpdate, self).get_password_prompts() d.update({ + r'Username for.*:': 'scm_username', + r'Password for.*:': 'scm_password', # FIXME: Configure whether we should auto accept host keys? r'Are you sure you want to continue connecting \(yes/no\)\?': 'yes', }) @@ -445,17 +497,31 @@ class RunProjectUpdate(BaseTask): ''' Hook for checking project update before running. ''' - if not super(RunProjectUpdate, self).pre_run_check(project_update, **kwargs): - return False - # FIXME: Check if project update is blocked by any jobs that are being run. - project = project_update.project - if project.jobs.filter(status__in=('pending', 'waiting', 'running')): - pass - return True + while True: + pk = project_update.pk + if project_update.status in ('pending', 'waiting'): + # Check if project update is blocked by any jobs or other + # updates that are active. Exclude job that is waiting for + # this project update. + project = project_update.project + jobs_qs = project.jobs.filter(status__in=('pending', 'running')) + pu_qs = project.project_updates.filter(status__in=('pending', 'running')) + pu_qs = pu_qs.exclude(pk=project_update.pk) + if jobs_qs.count() or pu_qs.count(): + #print 'project update %d waiting on' % pk, jobs_qs, pu_qs + project_update = self.update_model(pk, status='waiting') + time.sleep(4.0) + else: + project_update = self.update_model(pk, status='pending') + return True + elif project_update.cancel_flag: + project_update = self.update_model(pk, status='canceled') + return False + else: + return False def post_run_hook(self, project_update): ''' Hook for actions after project_update has completed. ''' # Start any jobs waiting on this update to finish. - diff --git a/awx/main/tests/projects.py b/awx/main/tests/projects.py index a215e8cd40..76cfb312d2 100644 --- a/awx/main/tests/projects.py +++ b/awx/main/tests/projects.py @@ -15,6 +15,7 @@ import django.test from django.test.client import Client from django.core.urlresolvers import reverse from django.test.utils import override_settings +from django.utils.timezone import now # AWX from awx.main.models import * @@ -902,3 +903,121 @@ class ProjectUpdatesTest(BaseTransactionTest): with self.current_user(self.super_django_user): response = self.post(url, {'scm_key_unlock': TEST_SSH_KEY_DATA_UNLOCK}, expect=202) + def create_test_job_template(self, **kwargs): + opts = { + 'name': 'test-job-template %s' % str(now()), + 'inventory': self.inventory, + 'project': self.project, + 'credential': self.credential, + 'job_type': 'run', + } + try: + opts['playbook'] = self.project.playbooks[0] + except (AttributeError, IndexError): + pass + opts.update(kwargs) + self.job_template = JobTemplate.objects.create(**opts) + return self.job_template + + def create_test_job(self, **kwargs): + job_template = kwargs.pop('job_template', None) + if job_template: + self.job = job_template.create_job(**kwargs) + else: + opts = { + 'name': 'test-job %s' % str(now()), + 'inventory': self.inventory, + 'project': self.project, + 'credential': self.credential, + 'job_type': 'run', + } + try: + opts['playbook'] = self.project.playbooks[0] + except (AttributeError, IndexError): + pass + opts.update(kwargs) + self.job = Job.objects.create(**opts) + return self.job + + def test_update_on_launch(self): + scm_url = getattr(settings, 'TEST_GIT_PUBLIC_HTTPS', + 'https://github.com/ansible/ansible.github.com.git') + if not all([scm_url]): + self.skipTest('no public git repo defined for https!') + self.organization = self.make_organizations(self.super_django_user, 1)[0] + self.inventory = Inventory.objects.create(name='test-inventory', + description='description for test-inventory', + organization=self.organization) + self.host = self.inventory.hosts.create(name='host.example.com', + inventory=self.inventory) + self.group = self.inventory.groups.create(name='test-group', + inventory=self.inventory) + self.group.hosts.add(self.host) + self.credential = Credential.objects.create(name='test-creds', + user=self.super_django_user) + self.project = self.create_project( + name='my public git project over https', + scm_type='git', + scm_url=scm_url, + scm_update_on_launch=True, + ) + self.check_project_update(self.project) + self.assertEqual(self.project.project_updates.count(), 1) + job_template = self.create_test_job_template() + job = self.create_test_job(job_template=job_template) + self.assertEqual(job.status, 'new') + self.assertFalse(job.passwords_needed_to_start) + self.assertTrue(job.start()) + self.assertEqual(job.status, 'pending') + job = Job.objects.get(pk=job.pk) + self.assertTrue(job.status in ('successful', 'failed')) + self.assertEqual(self.project.project_updates.count(), 2) + + def test_update_on_launch_with_project_passwords(self): + scm_url = getattr(settings, 'TEST_GIT_PRIVATE_HTTPS', '') + scm_username = getattr(settings, 'TEST_GIT_USERNAME', '') + scm_password = getattr(settings, 'TEST_GIT_PASSWORD', '') + if not all([scm_url, scm_username, scm_password]): + self.skipTest('no private git repo defined for https!') + self.organization = self.make_organizations(self.super_django_user, 1)[0] + self.inventory = Inventory.objects.create(name='test-inventory', + description='description for test-inventory', + organization=self.organization) + self.host = self.inventory.hosts.create(name='host.example.com', + inventory=self.inventory) + self.group = self.inventory.groups.create(name='test-group', + inventory=self.inventory) + self.group.hosts.add(self.host) + self.credential = Credential.objects.create(name='test-creds', + user=self.super_django_user) + self.project = self.create_project( + name='my private git project over https', + scm_type='git', + scm_url=scm_url, + scm_username=scm_username, + scm_password='ASK', + scm_update_on_launch=True, + ) + self.check_project_update(self.project, scm_password=scm_password) + self.assertEqual(self.project.project_updates.count(), 1) + job_template = self.create_test_job_template() + job = self.create_test_job(job_template=job_template) + self.assertEqual(job.status, 'new') + self.assertTrue(job.passwords_needed_to_start) + self.assertTrue('scm_password' in job.passwords_needed_to_start) + self.assertTrue(job.start(**{'scm_password': scm_password})) + self.assertEqual(job.status, 'pending') + job = Job.objects.get(pk=job.pk) + self.assertTrue(job.status in ('successful', 'failed')) + self.assertEqual(self.project.project_updates.count(), 2) + # Try again but with a bad password - the job should flag an error + # because the project update failed. + job = self.create_test_job(job_template=job_template) + self.assertEqual(job.status, 'new') + self.assertTrue(job.passwords_needed_to_start) + self.assertTrue('scm_password' in job.passwords_needed_to_start) + self.assertTrue(job.start(**{'scm_password': 'lasdkfjlsdkfj'})) + self.assertEqual(job.status, 'pending') + job = Job.objects.get(pk=job.pk) + self.assertEqual(job.status, 'error') + self.assertEqual(self.project.project_updates.count(), 3)