1
0
mirror of https://github.com/ansible/awx.git synced 2024-10-27 00:55:06 +03:00

Initial working version of job template callback with a test.

This commit is contained in:
Chris Church 2013-07-11 13:27:49 -04:00
parent 552e43668c
commit ca949eb71e
9 changed files with 328 additions and 58 deletions

View File

@ -1,7 +1,9 @@
[run]
source = ansibleworks
source = awx
branch = True
omit = ansibleworks/main/migrations/*
omit =
awx/main/migrations/*
awx/lib/site-packages/*
[report]
# Regexes for lines to exclude from consideration

View File

@ -412,6 +412,14 @@ class Credential(CommonModelNameNotUnique):
@property
def needs_sudo_password(self):
return self.sudo_password == 'ASK'
@property
def passwords_needed(self):
needed = []
for field in ('ssh_password', 'sudo_password', 'ssh_key_unlock'):
if getattr(self, 'needs_%s' % field):
needed.append(field)
return needed
def get_absolute_url(self):
return reverse('main:credential_detail', args=(self.pk,))
@ -646,14 +654,11 @@ class JobTemplate(CommonModel):
return reverse('main:job_template_detail', args=(self.pk,))
def can_start_without_user_input(self):
'''Return whether job template can be used to start a new job without
requiring any user input.'''
if not self.credential:
return False
for field in ('ssh_password', 'sudo_password', 'ssh_key_unlock'):
if getattr(self.credential, 'needs_%s' % field):
return False
return True
'''
Return whether job template can be used to start a new job without
requiring any user input.
'''
return bool(self.credential and not self.credential.passwords_needed)
class Job(CommonModel):
'''
@ -832,11 +837,7 @@ class Job(CommonModel):
def get_passwords_needed_to_start(self):
'''Return list of password field names needed to start the job.'''
needed = []
for field in ('ssh_password', 'sudo_password', 'ssh_key_unlock'):
if self.credential and getattr(self.credential, 'needs_%s' % field):
needed.append(field)
return needed
return (self.credential and self.credential.passwords_needed) or []
@property
def can_start(self):

View File

@ -127,36 +127,46 @@ class JobTemplateCallbackPermission(CustomRbac):
def has_permission(self, request, view, obj=None):
# If another authentication method was used and it's not a POST, return
# True to fall through to the next permission class.
if request.user or request.auth and request.method.lower() != 'post':
if (request.user or request.auth) and request.method.lower() != 'post':
return super(JobTemplateCallbackPermission, self).has_permission(request, view, obj)
return False
# FIXME
#try:
# job_template = JobTemplate.objects.get(active=True, pk=int(request.auth.split('-')[0]))
#except Job.DoesNotExist:
# return False
# Require method to be POST, host_config_key to be specified and match
# the requested job template, and require the job template to be
# active in order to proceed.
host_config_key = request.DATA.get('host_config_key', '')
if request.method.lower() != 'post':
return False
elif not host_config_key:
return False
elif obj and not obj.active:
return False
elif obj and obj.host_config_key != host_config_key:
return False
else:
return True
class JobTaskPermission(CustomRbac):
def has_permission(self, request, view, obj=None):
# If another authentication method was used other than the one for job
# callbacks, return True to fall through to the next permission class.
# callbacks, default to the superclass permissions checking.
if request.user or not request.auth:
return super(JobTaskPermission, self).has_permission(request, view, obj)
# FIXME: Verify that inventory or job event requested are for the same
# job ID present in the auth token, etc.
#try:
# job = Job.objects.get(active=True, status='running', pk=int(request.auth.split('-')[0]))
#except Job.DoesNotExist:
# return False
# Verify that the job ID present in the auth token is for a valid,
# active job.
try:
job = Job.objects.get(active=True, status='running',
pk=int(request.auth.split('-')[0]))
except (Job.DoesNotExist, TypeError):
return False
# Verify that the request method is one of those allowed for the given
# view, also that the job or inventory being accessed matches the auth
# token.
if view.model == Inventory and request.method.lower() in ('head', 'get'):
return True
return bool(not obj or obj.pk == job.inventory.pk)
elif view.model == JobEvent and request.method.lower() == 'post':
return True
return bool(not obj or obj.pk == job.pk)
else:
return False

View File

@ -38,6 +38,7 @@ class RunJob(Task):
if field == 'status':
update_fields.append('failed')
job.save(update_fields=update_fields)
# FIXME: Commit transaction?
return job
def get_path_to(self, *args):

View File

@ -145,7 +145,7 @@ class BaseTestMixin(object):
return ('random', 'combination')
def _generic_rest(self, url, data=None, expect=204, auth=None, method=None,
data_type=None, accept=None):
data_type=None, accept=None, remote_addr=None):
assert method is not None
method_name = method.lower()
if method_name not in ('options', 'head', 'get', 'delete'):
@ -153,6 +153,8 @@ class BaseTestMixin(object):
client_kwargs = {}
if accept:
client_kwargs['HTTP_ACCEPT'] = accept
if remote_addr:
client_kwargs['REMOTE_ADDR'] = remote_addr
client = Client(**client_kwargs)
auth = auth or self._current_auth
if auth:
@ -191,39 +193,46 @@ class BaseTestMixin(object):
else:
return None
def options(self, url, expect=200, auth=None, accept=None):
def options(self, url, expect=200, auth=None, accept=None,
remote_addr=None):
return self._generic_rest(url, data=None, expect=expect, auth=auth,
method='options', accept=accept)
method='options', accept=accept,
remote_addr=remote_addr)
def head(self, url, expect=200, auth=None, accept=None):
def head(self, url, expect=200, auth=None, accept=None, remote_addr=None):
return self._generic_rest(url, data=None, expect=expect, auth=auth,
method='head', accept=accept)
method='head', accept=accept,
remote_addr=remote_addr)
def get(self, url, expect=200, auth=None, accept=None):
def get(self, url, expect=200, auth=None, accept=None, remote_addr=None):
return self._generic_rest(url, data=None, expect=expect, auth=auth,
method='get', accept=accept)
method='get', accept=accept,
remote_addr=remote_addr)
def post(self, url, data, expect=204, auth=None, data_type=None,
accept=None):
accept=None, remote_addr=None):
return self._generic_rest(url, data=data, expect=expect, auth=auth,
method='post', data_type=data_type,
accept=accept)
accept=accept,
remote_addr=remote_addr)
def put(self, url, data, expect=200, auth=None, data_type=None,
accept=None):
accept=None, remote_addr=None):
return self._generic_rest(url, data=data, expect=expect, auth=auth,
method='put', data_type=data_type,
accept=accept)
accept=accept, remote_addr=remote_addr)
def patch(self, url, data, expect=200, auth=None, data_type=None,
accept=None):
accept=None, remote_addr=None):
return self._generic_rest(url, data=data, expect=expect, auth=auth,
method='patch', data_type=data_type,
accept=accept)
accept=accept, remote_addr=remote_addr)
def delete(self, url, expect=201, auth=None, data_type=None, accept=None):
def delete(self, url, expect=201, auth=None, data_type=None, accept=None,
remote_addr=None):
return self._generic_rest(url, data=None, expect=expect, auth=auth,
method='delete', accept=accept)
method='delete', accept=accept,
remote_addr=remote_addr)
def get_urls(self, collection_url, auth=None):
# TODO: this test helper function doesn't support pagination

View File

@ -1,18 +1,27 @@
# Copyright (c) 2013 AnsibleWorks, Inc.
# All Rights Reserved.
# Python
import datetime
import json
import socket
import struct
import uuid
# Django
from django.contrib.auth.models import User as DjangoUser
from django.core.urlresolvers import reverse
from django.db import transaction
import django.test
from django.test.client import Client
from django.test.utils import override_settings
# AWX
from awx.main.models import *
from awx.main.tests.base import BaseTestMixin
__all__ = ['JobTemplateTest', 'JobTest', 'JobStartCancelTest']
__all__ = ['JobTemplateTest', 'JobTest', 'JobStartCancelTest',
'JobTemplateCallbackTest']
TEST_PLAYBOOK = '''- hosts: all
gather_facts: false
@ -308,6 +317,7 @@ class BaseJobTestMixin(BaseTestMixin):
inventory= self.inv_eng,
project=self.proj_dev,
playbook=self.proj_dev.playbooks[0],
host_config_key=uuid.uuid4().hex,
created_by=self.user_sue,
)
self.job_eng_check = self.jt_eng_check.create_job(
@ -320,6 +330,7 @@ class BaseJobTestMixin(BaseTestMixin):
inventory= self.inv_eng,
project=self.proj_dev,
playbook=self.proj_dev.playbooks[0],
host_config_key=uuid.uuid4().hex,
created_by=self.user_sue,
)
self.job_eng_run = self.jt_eng_run.create_job(
@ -335,6 +346,7 @@ class BaseJobTestMixin(BaseTestMixin):
inventory= self.inv_sup,
project=self.proj_test,
playbook=self.proj_test.playbooks[0],
host_config_key=uuid.uuid4().hex,
created_by=self.user_sue,
)
self.job_sup_check = self.jt_sup_check.create_job(
@ -347,6 +359,7 @@ class BaseJobTestMixin(BaseTestMixin):
inventory= self.inv_sup,
project=self.proj_test,
playbook=self.proj_test.playbooks[0],
host_config_key=uuid.uuid4().hex,
created_by=self.user_sue,
)
self.job_sup_run = self.jt_sup_run.create_job(
@ -363,6 +376,7 @@ class BaseJobTestMixin(BaseTestMixin):
project=self.proj_prod,
playbook=self.proj_prod.playbooks[0],
credential=self.cred_ops_east,
host_config_key=uuid.uuid4().hex,
created_by=self.user_sue,
)
self.job_ops_east_check = self.jt_ops_east_check.create_job(
@ -375,6 +389,7 @@ class BaseJobTestMixin(BaseTestMixin):
project=self.proj_prod,
playbook=self.proj_prod.playbooks[0],
credential=self.cred_ops_east,
host_config_key=uuid.uuid4().hex,
created_by=self.user_sue,
)
self.job_ops_east_run = self.jt_ops_east_run.create_job(
@ -387,6 +402,7 @@ class BaseJobTestMixin(BaseTestMixin):
project=self.proj_prod,
playbook=self.proj_prod.playbooks[0],
credential=self.cred_ops_west,
host_config_key=uuid.uuid4().hex,
created_by=self.user_sue,
)
self.job_ops_west_check = self.jt_ops_west_check.create_job(
@ -399,6 +415,7 @@ class BaseJobTestMixin(BaseTestMixin):
project=self.proj_prod,
playbook=self.proj_prod.playbooks[0],
credential=self.cred_ops_west,
host_config_key=uuid.uuid4().hex,
created_by=self.user_sue,
)
self.job_ops_west_run = self.jt_ops_west_run.create_job(
@ -965,3 +982,142 @@ class JobStartCancelTest(BaseJobTestMixin, django.test.LiveServerTestCase):
self.assertTrue(qs.count())
self.check_pagination_and_size(response, qs.count())
self.check_list_ids(response, qs)
@override_settings(CELERY_ALWAYS_EAGER=True,
CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
ANSIBLE_TRANSPORT='local',
MIDDLEWARE_CLASSES=MIDDLEWARE_CLASSES)
class JobTemplateCallbackTest(BaseJobTestMixin, django.test.LiveServerTestCase):
'''Job template callback tests for empheral hosts.'''
def setUp(self):
super(JobTemplateCallbackTest, self).setUp()
settings.INTERNAL_API_URL = self.live_server_url
# Monkeypatch socket module DNS lookup functions for testing.
self._original_gethostbyaddr = socket.gethostbyaddr
self._original_getaddrinfo = socket.getaddrinfo
socket.gethostbyaddr = self.gethostbyaddr
socket.getaddrinfo = self.getaddrinfo
def tearDown(self):
super(JobTemplateCallbackTest, self).tearDown()
socket.gethostbyaddr = self._original_gethostbyaddr
socket.getaddrinfo = self._original_getaddrinfo
def atoh(self, a):
'''Convert IP address to integer in host byte order.'''
return socket.ntohl(struct.unpack('I', socket.inet_aton(a))[0])
def htoa(self, n):
'''Convert integer in host byte order to IP address.'''
return socket.inet_ntoa(struct.pack('I', socket.htonl(n)))
def get_test_ips_for_host(self, host):
'''Return test IP address(es) for given test hostname.'''
ips = []
try:
h = Host.objects.get(name=host)
# Primary IP for host (both forward/reverse lookups work).
val = self.atoh('127.10.0.0') + h.pk
ips.append(self.htoa(val))
# Secondary IP for host (both forward/reverse lookups work).
if h.pk % 2 == 0:
val = self.atoh('127.20.0.0') + h.pk
ips.append(self.htoa(val))
# Additional IP for host (only forward lookups work).
if h.pk % 3 == 0:
val = self.atoh('127.30.0.0') + h.pk
ips.append(self.htoa(val))
except Host.DoesNotExist:
pass
return ips
def get_test_host_for_ip(self, ip):
'''Return test hostname for given test IP address.'''
if not ip.startswith('127.10.') and not ip.startswith('127.20.'):
return None
val = self.atoh(ip)
try:
return Host.objects.get(pk=(val & 0x0ffff)).name
except Host.DoesNotExist:
return None
def test_dummy_host_ip_lookup(self):
all_ips = set()
for host in Host.objects.all():
ips = self.get_test_ips_for_host(host.name)
#print host, ips
self.assertTrue(ips)
all_ips.update(ips)
ips = self.get_test_ips_for_host('invalid_host_name')
self.assertFalse(ips)
for ip in all_ips:
host = self.get_test_host_for_ip(ip)
#print ip, host
if ip.startswith('127.30.'):
continue
self.assertTrue(host)
ips = self.get_test_ips_for_host(host)
self.assertTrue(ip in ips)
host = self.get_test_host_for_ip('127.10.254.254')
self.assertFalse(host)
def gethostbyaddr(self, ip):
#print 'gethostbyaddr', ip
if not ip.startswith('127.'):
return self._original_gethostbyaddr(ip)
host = self.get_test_host_for_ip(ip)
if not host:
raise socket.herror('unknown test host')
raddr = '.'.join(list(reversed(ip.split('.'))) + ['in-addr', 'arpa'])
return (host, [raddr], [ip])
def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
#print 'getaddrinfo', host, port, family, socktype, proto, flags
if family or socktype or proto or flags:
return self._original_getaddrinfo(host, port, family, socktype,
proto, flags)
port = port or 0
try:
socket.inet_aton(host)
addrs = [host]
except socket.error:
addrs = self.get_test_ips_for_host(host)
if not addrs:
raise socket.gaierror('test host not found')
results = []
for addr in addrs:
results.append((socket.AF_INET, socket.SOCK_STREAM,
socket.IPPROTO_TCP, '', (addr, port)))
results.append((socket.AF_INET, socket.SOCK_DGRAM,
socket.IPPROTO_UDP, '', (addr, port)))
return results
def test_job_template_callback(self):
# Find a valid job template to use to test the callback.
job_template = None
qs = JobTemplate.objects.filter(job_type='run',
credential__isnull=False)
qs = qs.exclude(host_config_key='')
for jt in qs:
if not jt.can_start_without_user_input():
continue
job_template = jt
break
self.assertTrue(job_template)
url = reverse('main:job_template_callback', args=(job_template.pk,))
# Test a POST to start a new job.
with self.current_user(None):
data = dict(host_config_key=job_template.host_config_key)
host = job_template.inventory.hosts.order_by('-pk')[0]
ip = self.get_test_ips_for_host(host.name)[0]
jobs_qs = job_template.jobs.filter(launch_type='callback')
self.assertEqual(jobs_qs.count(), 0)
self.post(url, data, expect=202, remote_addr=ip)
self.assertEqual(jobs_qs.count(), 1)
job = jobs_qs[0]
self.assertEqual(job.launch_type, 'callback')
self.assertEqual(job.limit, host.name)
self.assertEqual(job.hosts.count(), 1)
self.assertEqual(job.hosts.all()[0], host)

View File

@ -4,6 +4,7 @@
# Python
import datetime
import re
import socket
import sys
# Django
@ -1070,27 +1071,107 @@ class JobTemplateCallback(generics.RetrieveAPIView):
'''
Configure a host to POST to this resource using the `host_config_key`.
'''
model = JobTemplate
permission_classes = (JobTemplateCallbackPermission,)
def get(self, request, *args, **kwargs):
def find_host(self):
'''
Find the host in the job template's inventory that matches the remote
host for the current request.
'''
# Find the list of remote host names/IPs to check.
remote_hosts = set()
for header in settings.REMOTE_HOST_HEADERS:
value = self.request.META.get(header, '').strip()
if value:
remote_hosts.add(value)
# Add the reverse lookup of IP addresses.
for rh in list(remote_hosts):
try:
result = socket.gethostbyaddr(rh)
except socket.herror:
continue
remote_hosts.add(result[0])
remote_hosts.update(result[1])
# Filter out any .arpa results.
for rh in list(remote_hosts):
if rh.endswith('.arpa'):
remote_hosts.remove(rh)
if not remote_hosts:
return
# Find the host objects to search for a match.
obj = self.get_object()
qs = obj.inventory.hosts.filter(active=True)
# First try for an exact match on the name.
try:
return qs.get(name__in=remote_hosts)
except (Host.DoesNotExist, Host.MultipleObjectsReturned):
pass
# Next, try matching based on name or ansible_ssh_host variable.
matches = dict()
for host in qs:
ansible_ssh_host = host.variables_dict.get('ansible_ssh_host', '')
if ansible_ssh_host in remote_hosts:
if host not in matches:
matches[host] = 0
matches[host] += 2
if host.name != ansible_ssh_host and host.name in remote_hosts:
if host not in matches:
matches[host] = 0
matches[host] += 1
if len(matches) == 1:
return matches.keys()[0]
# Try to resolve forward addresses for each host to find a match.
for host in qs:
hostnames = set([host.name])
ansible_ssh_host = host.variables_dict.get('ansible_ssh_host', '')
if ansible_ssh_host:
hostnames.add(ansible_ssh_host)
for hostname in hostnames:
try:
result = socket.getaddrinfo(hostname, None)
possible_ips = set(x[4][0] for x in result)
possible_ips.discard(hostname)
if possible_ips and possible_ips & remote_hosts:
if host in matches:
matches[host] += 1
else:
matches[host] = 1
except socket.gaierror:
pass
# Return the host with the highest match weight (in case of multiple
# matches).
if matches:
return sorted(matches.items(), key=lambda x: x[1])[-1][0]
def get(self, request, *args, **kwargs):
job_template = self.get_object()
data = dict(
host_config_key=obj.host_config_key,
host_config_key=job_template.host_config_key,
matched_host=getattr(self.find_host(), 'name', None),
)
if settings.DEBUG:
d = dict([(k,v) for k,v in request.META.items()
if k.startswith('HTTP_') or k.startswith('REMOTE_')])
data['request_meta'] = d
return Response(data)
def post(self, request, *args, **kwargs):
obj = self.get_object()
job_template = self.get_object()
# Permission class should have already validated host_config_key.
# FIXME: Find host from request.
limit = obj.limit
# FIXME: Update limit based on host.
job = obj.create_job(limit=limit, launch_type='callback')
host = self.find_host()
if not host:
data = dict(msg='No matching host could be found!')
return Response(data, status=400)
if not job_template.can_start_without_user_input():
data = dict(msg='Cannot start automatically, user input required!')
return Response(data, status=400)
limit = ':'.join(filter(None, [job_template.limit, host.name]))
job = job_template.create_job(limit=limit, launch_type='callback')
result = job.start()
if not result:
data = dict(passwords_needed_to_start=job.get_passwords_needed_to_start())
data = dict(msg='Error starting job!')
return Response(data, status=400)
else:
return Response(status=202)

View File

@ -92,6 +92,11 @@ SECRET_KEY = 'p7z7g1ql4%6+(6nlebb6hdk7sd^&fnjpal308%n%+p^_e6vo1y'
# See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []
# HTTP headers and meta keys to search to determine remote host name or IP. Add
# additional items to this list, such as "HTTP_X_FORWARDED_FOR", if behind a
# reverse proxy.
REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST']
TEMPLATE_CONTEXT_PROCESSORS += (
'django.core.context_processors.request',
'awx.ui.context_processors.settings',

View File

@ -55,6 +55,11 @@ LANGUAGE_CODE = 'en-us'
# the secret key from an environment variable or a file instead.
SECRET_KEY = 'p7z7g1ql4%6+(6nlebb6hdk7sd^&fnjpal308%n%+p^_e6vo1y'
# HTTP headers and meta keys to search to determine remote host name or IP. Add
# additional items to this list, such as "HTTP_X_FORWARDED_FOR", if behind a
# reverse proxy.
REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST']
# Email address that error messages come from.
SERVER_EMAIL = 'root@localhost'