diff --git a/awx/main/serializers.py b/awx/main/serializers.py index 3add7f2628..4ef1793bf6 100644 --- a/awx/main/serializers.py +++ b/awx/main/serializers.py @@ -25,6 +25,7 @@ from rest_framework import serializers # AWX from awx.main.models import * +from awx.main.utils import update_scm_url BASE_FIELDS = ('id', 'url', 'related', 'summary_fields', 'created', 'modified', 'name', 'description') @@ -299,11 +300,26 @@ class ProjectSerializer(BaseSerializer): def validate_scm_url(self, attrs, source): if self.object: - scm_type = attrs.get('scm_type', self.object.scm_type) + scm_type = attrs.get('scm_type', self.object.scm_type) or '' + scm_username = attrs.get('scm_username', self.object.scm_username) or '' + scm_password = attrs.get('scm_password', self.object.scm_password) or '' else: - scm_type = attrs.get('scm_type', '') + scm_type = attrs.get('scm_type', '') or '' + scm_username = attrs.get('scm_username', '') or '' + scm_password = attrs.get('scm_password', '') or '' scm_url = unicode(attrs.get(source, None) or '') + try: + if scm_username and scm_password: + scm_url = update_scm_url(scm_type, scm_url, scm_username, + '********') + elif scm_username: + scm_url = update_scm_url(scm_type, scm_url, scm_username) + else: + scm_url = update_scm_url(scm_type, scm_url) + except ValueError, e: + raise serializers.ValidationError((e.args or ('Invalid SCM URL',))[0]) scm_url_parts = urlparse.urlsplit(scm_url) + print scm_url_parts if scm_type and not any(scm_url_parts): raise serializers.ValidationError('SCM URL must be provided') return attrs diff --git a/awx/main/tasks.py b/awx/main/tasks.py index a7d4e421e9..64a11a0ceb 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -40,7 +40,6 @@ class BaseTask(Task): name = None model = None - idle_timeout = None def update_model(self, pk, **updates): ''' @@ -129,6 +128,9 @@ class BaseTask(Task): def build_output_replacements(self, instance, **kwargs): return [] + def get_idle_timeout(self): + return None + def get_password_prompts(self): ''' Return a dictionary of prompt regular expressions and password lookup @@ -152,6 +154,7 @@ class BaseTask(Task): child.logfile_read = logfile canceled = False last_stdout_update = time.time() + idle_timeout = self.get_idle_timeout() expect_list = [] expect_passwords = {} for n, item in enumerate(self.get_password_prompts().items()): @@ -174,7 +177,7 @@ class BaseTask(Task): canceled = True # FIXME: Configurable idle timeout? Find a way to determine if task # is hung waiting at a prompt. - if self.idle_timeout and (time.time() - last_stdout_update) > self.idle_timeout: + if idle_timeout and (time.time() - last_stdout_update) > idle_timeout: child.close(True) canceled = True if canceled: @@ -430,7 +433,6 @@ class RunProjectUpdate(BaseTask): name = 'run_project_update' model = ProjectUpdate - #idle_timeout = 30 def build_passwords(self, project_update, **kwargs): ''' @@ -463,27 +465,33 @@ class RunProjectUpdate(BaseTask): Helper method to build SCM url and extra vars with parameters needed for authentication. ''' - # FIXME: May need to pull username/password out of URL in other cases. extra_vars = {} project = project_update.project scm_type = project.scm_type - scm_url = project.scm_url + scm_url = update_scm_url(scm_type, project.scm_url) + scm_url_parts = urlparse.urlsplit(scm_url) scm_username = kwargs.get('passwords', {}).get('scm_username', '') + scm_username = scm_username or scm_url_parts.username or '' scm_password = kwargs.get('passwords', {}).get('scm_password', '') + scm_password = scm_password or scm_url_parts.password or '' if scm_username and scm_password not in ('ASK', ''): if scm_type == 'svn': + # FIXME: Need to somehow escape single/double quotes in username/password extra_vars['scm_username'] = scm_username extra_vars['scm_password'] = scm_password + scm_url = update_scm_url(scm_type, scm_url, False, False) + elif scm_url_parts.scheme == 'ssh': + scm_url = update_scm_url(scm_type, scm_url, scm_username, False) else: scm_url = update_scm_url(scm_type, scm_url, scm_username, scm_password) elif scm_username: if scm_type == 'svn': extra_vars['scm_username'] = scm_username + extra_vars['scm_password'] = '' + scm_url = update_scm_url(scm_type, scm_url, False, False) else: - scm_url = update_scm_url(scm_type, scm_url, scm_username) - else: - scm_url = update_scm_url(scm_type, scm_url) + scm_url = update_scm_url(scm_type, scm_url, scm_username, False) return scm_url, extra_vars def build_args(self, project_update, **kwargs): @@ -506,6 +514,7 @@ class RunProjectUpdate(BaseTask): 'scm_clean': project.scm_clean, 'scm_delete_on_update': scm_delete_on_update, }) + #print extra_vars args.extend(['-e', json.dumps(extra_vars)]) args.append('project_update.yml') @@ -568,11 +577,15 @@ class RunProjectUpdate(BaseTask): d.update({ r'Username for.*:': 'scm_username', r'Password for.*:': 'scm_password', + r'^Password:\s*?$': 'scm_password', # SSH prompt for git. # FIXME: Configure whether we should auto accept host keys? r'Are you sure you want to continue connecting \(yes/no\)\?': 'yes', }) return d + def get_idle_timeout(self): + return getattr(settings, 'PROJECT_UPDATE_IDLE_TIMEOUT', None) + def pre_run_check(self, project_update, **kwargs): ''' Hook for checking project update before running. diff --git a/awx/main/tests/projects.py b/awx/main/tests/projects.py index 85d89ef0ab..aab998dece 100644 --- a/awx/main/tests/projects.py +++ b/awx/main/tests/projects.py @@ -3,6 +3,7 @@ # Python import datetime +import getpass import json import os import re @@ -620,7 +621,8 @@ class ProjectsTest(BaseTest): @override_settings(CELERY_ALWAYS_EAGER=True, CELERY_EAGER_PROPAGATES_EXCEPTIONS=True, - ANSIBLE_TRANSPORT='local') + ANSIBLE_TRANSPORT='local', + PROJECT_UPDATE_IDLE_TIMEOUT=30) class ProjectUpdatesTest(BaseTransactionTest): def setUp(self): @@ -802,6 +804,29 @@ class ProjectUpdatesTest(BaseTransactionTest): password='testpass') self.assertEqual(new_url_up, updated_url) + def is_public_key_in_authorized_keys(self): + auth_keys = set() + auth_keys_path = os.path.expanduser('~/.ssh/authorized_keys') + if os.path.exists(auth_keys_path): + for line in file(auth_keys_path, 'r'): + if line.strip(): + key = tuple(line.strip().split()[:2]) + auth_keys.add(key) + pub_keys = set() + rsa_key_path = os.path.expanduser('~/.ssh/id_rsa.pub') + if os.path.exists(rsa_key_path): + for line in file(rsa_key_path, 'r'): + if line.strip(): + key = tuple(line.strip().split()[:2]) + pub_keys.add(key) + dsa_key_path = os.path.expanduser('~/.ssh/id_dsa.pub') + if os.path.exists(dsa_key_path): + for line in file(dsa_key_path, 'r'): + if line.strip(): + key = tuple(line.strip().split()[:2]) + pub_keys.add(key) + return bool(auth_keys & pub_keys) + def check_project_update(self, project, should_fail=False, **kwargs): pu = kwargs.pop('project_update', None) if not pu: @@ -821,6 +846,9 @@ class ProjectUpdatesTest(BaseTransactionTest): self.assertTrue(match, pu.job_args) scm_url_in_args = match.groups()[0] self.assertFalse(scm_url_in_args.startswith('/'), scm_url_in_args) + #return pu + # Make sure scm_password doesn't show up anywhere in args or output + # from project update. scm_password = kwargs.get('scm_password', decrypt_field(project, 'scm_password')) if scm_password not in ('', 'ASK'): @@ -831,6 +859,8 @@ class ProjectUpdatesTest(BaseTransactionTest): pu.result_stdout) self.assertFalse(scm_password in pu.result_traceback, pu.result_traceback) + # Make sure scm_key_unlock doesn't show up anywhere in args or output + # from project update. scm_key_unlock = kwargs.get('scm_key_unlock', decrypt_field(project, 'scm_key_unlock')) if scm_key_unlock not in ('', 'ASK'): @@ -865,10 +895,14 @@ class ProjectUpdatesTest(BaseTransactionTest): self.fail('no file found to change!') def check_project_scm(self, project): + project = Project.objects.get(pk=project.pk) project_path = project.get_project_path(check_if_exists=False) # If project could be auto-updated on creation, the project dir should # already exist, otherwise run an initial checkout. if project.scm_type and not project.scm_passwords_needed: + self.assertTrue(project.last_update) + self.check_project_update(project, + project_udpate=project.last_update) self.assertTrue(os.path.exists(project_path)) else: self.assertFalse(os.path.exists(project_path)) @@ -931,13 +965,18 @@ class ProjectUpdatesTest(BaseTransactionTest): self.assertFalse(os.path.exists(untracked_path)) # Change username/password for private projects and verify the update # fails (but doesn't cause the task to hang). + scm_url_parts = urlparse.urlsplit(project.scm_url) if project.scm_username and project.scm_password not in ('', 'ASK'): scm_username = project.scm_username + should_still_fail = not (getpass.getuser() == scm_username and + scm_url_parts.hostname == 'localhost' and + 'ssh' in scm_url_parts.scheme and + self.is_public_key_in_authorized_keys()) # Clear username only. project = Project.objects.get(pk=project.pk) project.scm_username = '' project.save() - self.check_project_update(project, should_fail=True) + self.check_project_update(project, should_fail=should_still_fail) # Try invalid username. project = Project.objects.get(pk=project.pk) project.scm_username = 'not a\\ valid\' user" name' @@ -948,17 +987,20 @@ class ProjectUpdatesTest(BaseTransactionTest): project.scm_username = '' project.scm_password = '' project.save() - self.check_project_update(project, should_fail=True) + self.check_project_update(project, should_fail=should_still_fail) # Set username, but no password. project = Project.objects.get(pk=project.pk) project.scm_username = scm_username project.save() - self.check_project_update(project, should_fail=True) + self.check_project_update(project, should_fail=should_still_fail) # Set username, with invalid password. project = Project.objects.get(pk=project.pk) project.scm_password = 'not a\\ valid\' "password' project.save() - self.check_project_update(project, should_fail=True) + if project.scm_type == 'svn': + self.check_project_update(project, should_fail=True)#should_still_fail) + else: + self.check_project_update(project, should_fail=should_still_fail) def test_public_git_project_over_https(self): scm_url = getattr(settings, 'TEST_GIT_PUBLIC_HTTPS', @@ -1025,8 +1067,7 @@ class ProjectUpdatesTest(BaseTransactionTest): ) self.check_project_update(project2, should_fail=True) - def test_git_project_from_local_path(self): - # Create temp repository directory. + def create_local_git_repo(self): repo_dir = tempfile.mkdtemp() self._temp_project_dirs.append(repo_dir) handle, playbook_path = tempfile.mkstemp(suffix='.yml', dir=repo_dir) @@ -1040,7 +1081,10 @@ class ProjectUpdatesTest(BaseTransactionTest): stderr=subprocess.PIPE) subprocess.check_call(['git', 'commit', '-m', 'blah'], cwd=repo_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # Now test project using local repo. + return repo_dir + + def test_git_project_from_local_path(self): + repo_dir = self.create_local_git_repo() project = self.create_project( name='my git project from local path', scm_type='git', @@ -1048,6 +1092,22 @@ class ProjectUpdatesTest(BaseTransactionTest): ) self.check_project_scm(project) + def test_git_project_via_ssh_loopback(self): + scm_username = getattr(settings, 'TEST_SSH_LOOPBACK_USERNAME', '') + scm_password = getattr(settings, 'TEST_SSH_LOOPBACK_PASSWORD', '') + if not all([scm_username, scm_password]): + self.skipTest('no ssh loopback username/password defined!') + repo_dir = self.create_local_git_repo() + scm_url = 'ssh://localhost%s' % repo_dir + project = self.create_project( + name='my git project via ssh loopback', + scm_type='git', + scm_url=scm_url, + scm_username=scm_username, + scm_password=scm_password, + ) + self.check_project_scm(project) + def test_public_hg_project_over_https(self): scm_url = getattr(settings, 'TEST_HG_PUBLIC_HTTPS', 'https://bitbucket.org/cchurch/django-hotrunner') @@ -1091,9 +1151,7 @@ class ProjectUpdatesTest(BaseTransactionTest): def test_private_hg_project_over_ssh(self): scm_url = getattr(settings, 'TEST_HG_PRIVATE_SSH', '') scm_key_data = getattr(settings, 'TEST_HG_KEY_DATA', '') - scm_username = getattr(settings, 'TEST_HG_USERNAME', '') - scm_password = 'blahblahblah' - if not all([scm_url, scm_key_data, scm_username]): + if not all([scm_url, scm_key_data]): self.skipTest('no private hg repo defined for ssh!') project = self.create_project( name='my private hg project over ssh', @@ -1102,19 +1160,9 @@ class ProjectUpdatesTest(BaseTransactionTest): scm_key_data=scm_key_data, ) self.check_project_scm(project) - # Test project using SSH username/password instead of key. Should fail - # because of bad password, but never hang. - project2 = self.create_project( - name='my other private hg project over ssh', - scm_type='hg', - scm_url=scm_url, - scm_username=scm_username, - scm_password=scm_password, - ) - self.check_project_update(project2, should_fail=True) + # hg doesn't support password for ssh:// urls. - def test_hg_project_from_local_path(self): - # Create temp repository directory. + def create_local_hg_repo(self): repo_dir = tempfile.mkdtemp() self._temp_project_dirs.append(repo_dir) handle, playbook_path = tempfile.mkstemp(suffix='.yml', dir=repo_dir) @@ -1128,7 +1176,10 @@ class ProjectUpdatesTest(BaseTransactionTest): stderr=subprocess.PIPE) subprocess.check_call(['hg', 'commit', '-m', 'blah'], cwd=repo_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # Now test project using local repo. + return repo_dir + + def test_hg_project_from_local_path(self): + repo_dir = self.create_local_hg_repo() project = self.create_project( name='my hg project from local path', scm_type='hg', @@ -1136,6 +1187,23 @@ class ProjectUpdatesTest(BaseTransactionTest): ) self.check_project_scm(project) + def _test_hg_project_via_ssh_loopback(self): + # hg doesn't support password for ssh:// urls. + scm_username = getattr(settings, 'TEST_SSH_LOOPBACK_USERNAME', '') + if not all([scm_username]): + self.skipTest('no ssh loopback username defined!') + if not self.is_public_key_in_authorized_keys(): + self.skipTest('ssh loopback for hg requires public key in authorized keys') + repo_dir = self.create_local_hg_repo() + scm_url = 'ssh://localhost/%s' % repo_dir + project = self.create_project( + name='my hg project via ssh loopback', + scm_type='hg', + scm_url=scm_url, + scm_username=scm_username, + ) + self.check_project_scm(project) + def test_public_svn_project_over_https(self): scm_url = getattr(settings, 'TEST_SVN_PUBLIC_HTTPS', 'https://github.com/ansible/ansible.github.com') @@ -1163,8 +1231,7 @@ class ProjectUpdatesTest(BaseTransactionTest): ) self.check_project_scm(project) - def test_svn_project_from_local_path(self): - # Create temp repository directory. + def create_local_svn_repo(self): repo_dir = tempfile.mkdtemp() self._temp_project_dirs.append(repo_dir) subprocess.check_call(['svnadmin', 'create', '.'], cwd=repo_dir, @@ -1178,8 +1245,11 @@ class ProjectUpdatesTest(BaseTransactionTest): 'file://%s/%s' % (repo_dir, os.path.basename(playbook_path))], cwd=repo_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return repo_dir + + def test_svn_project_from_local_path(self): + repo_dir = self.create_local_svn_repo() scm_url = 'file://%s' % repo_dir - # Now test project using local repo. project = self.create_project( name='my svn project from local path', scm_type='svn', @@ -1187,6 +1257,22 @@ class ProjectUpdatesTest(BaseTransactionTest): ) self.check_project_scm(project) + def test_svn_project_via_ssh_loopback(self): + scm_username = getattr(settings, 'TEST_SSH_LOOPBACK_USERNAME', '') + scm_password = getattr(settings, 'TEST_SSH_LOOPBACK_PASSWORD', '') + if not all([scm_username, scm_password]): + self.skipTest('no ssh loopback username/password defined!') + repo_dir = self.create_local_svn_repo() + scm_url = 'svn+ssh://localhost%s' % repo_dir + project = self.create_project( + name='my svn project via ssh loopback', + scm_type='svn', + scm_url=scm_url, + scm_username=scm_username, + scm_password=scm_password, + ) + self.check_project_scm(project) + def test_prompt_for_scm_password_on_update(self): scm_url = getattr(settings, 'TEST_GIT_PUBLIC_HTTPS', 'https://github.com/ansible/ansible.github.com.git') diff --git a/awx/main/utils.py b/awx/main/utils.py index 60c755c322..6cd532bc6c 100644 --- a/awx/main/utils.py +++ b/awx/main/utils.py @@ -141,6 +141,8 @@ def update_scm_url(scm_type, url, username=True, password=True): # svn: http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.advanced.reposurls if scm_type not in ('git', 'hg', 'svn'): raise ValueError('unsupported SCM type "%s"' % str(scm_type)) + if not url.strip(): + return '' parts = urlparse.urlsplit(url) #print parts if '://' not in url: diff --git a/awx/settings/local_settings.py.example b/awx/settings/local_settings.py.example index 01b37dfd44..f4de165e69 100644 --- a/awx/settings/local_settings.py.example +++ b/awx/settings/local_settings.py.example @@ -260,6 +260,11 @@ TEST_SVN_PASSWORD = '' TEST_SVN_PUBLIC_HTTPS = 'https://github.com/ansible/ansible-examples' TEST_SVN_PRIVATE_HTTPS = 'https://github.com/ansible/ansible-doc' +# To test repo access via SSH login to localhost. +import getpass +TEST_SSH_LOOPBACK_USERNAME = getpass.getuser() +TEST_SSH_LOOPBACK_PASSWORD = '' + ############################################################################### # LDAP TEST SETTINGS ###############################################################################