shaba/awx
1
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2024-11-01 08:21:15 +03:00
Code Releases Activity

Merge pull request #4589 from AlanCoding/mah_galaxy

Browse Source 
Allow use of user-specified private galaxy server with fallback

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul
This commit is contained in:
softwarefactory-project-zuul[bot] 2019-10-09 15:01:12 +00:00 committed by GitHub
commit 0f1a92bd51
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 264 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
awx/main/conf.py
View File

@ -2,6 +2,7 @@
import json
import logging
import os
from distutils.version import LooseVersion as Version
# Django
from django.utils.translation import ugettext_lazy as _
@ -410,6 +411,15 @@ register(
category_slug='system',
)
register(
'PROJECT_UPDATE_VVV',
field_class=fields.BooleanField,
label=_('Run Project Updates With Higher Verbosity'),
help_text=_('Adds the CLI -vvv flag to ansible-playbook runs of project_update.yml used for project updates.'),
category=_('Jobs'),
category_slug='jobs',
)
register(
'AWX_ROLES_ENABLED',
field_class=fields.BooleanField,
@ -430,6 +440,75 @@ register(
category_slug='jobs',
)
register(
'PRIMARY_GALAXY_URL',
field_class=fields.URLField,
required=False,
allow_blank=True,
label=_('Primary Galaxy Server URL'),
help_text=_(
'For organizations that run their own Galaxy service, this gives the option to specify a '
'host as the primary galaxy server. Requirements will be downloaded from the primary if the '
'specific role or collection is available there. If the content is not avilable in the primary, '
'or if this field is left blank, it will default to galaxy.ansible.com.'
),
category=_('Jobs'),
category_slug='jobs'
)
register(
'PRIMARY_GALAXY_USERNAME',
field_class=fields.CharField,
required=False,
allow_blank=True,
label=_('Primary Galaxy Server Username'),
help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
'The username to use for basic authentication against the Galaxy instance, '
'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'),
category=_('Jobs'),
category_slug='jobs'
)
register(
'PRIMARY_GALAXY_PASSWORD',
field_class=fields.CharField,
encrypted=True,
required=False,
allow_blank=True,
label=_('Primary Galaxy Server Password'),
help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
'The password to use for basic authentication against the Galaxy instance, '
'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'),
category=_('Jobs'),
category_slug='jobs'
)
register(
'PRIMARY_GALAXY_TOKEN',
field_class=fields.CharField,
encrypted=True,
required=False,
allow_blank=True,
label=_('Primary Galaxy Server Token'),
help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
'The token to use for connecting with the Galaxy instance, '
'this is mutually exclusive with corresponding username and password settings.'),
category=_('Jobs'),
category_slug='jobs'
)
register(
'PRIMARY_GALAXY_AUTH_URL',
field_class=fields.CharField,
required=False,
allow_blank=True,
label=_('Primary Galaxy Authentication URL'),
help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
'The token_endpoint of a Keycloak server.'),
category=_('Jobs'),
category_slug='jobs'
)
register(
'STDOUT_MAX_BYTES_DISPLAY',
field_class=fields.IntegerField,
@ -700,4 +779,75 @@ def logging_validate(serializer, attrs):
return attrs
def galaxy_validate(serializer, attrs):
"""Ansible Galaxy config options have mutual exclusivity rules, these rules
are enforced here on serializer validation so that users will not be able
to save settings which obviously break all project updates.
"""
prefix = 'PRIMARY_GALAXY_'
from awx.main.constants import GALAXY_SERVER_FIELDS
if not any('{}{}'.format(prefix, subfield.upper()) in attrs for subfield in GALAXY_SERVER_FIELDS):
return attrs
def _new_value(setting_name):
if setting_name in attrs:
return attrs[setting_name]
elif not serializer.instance:
return ''
return getattr(serializer.instance, setting_name, '')
galaxy_data = {}
for subfield in GALAXY_SERVER_FIELDS:
galaxy_data[subfield] = _new_value('{}{}'.format(prefix, subfield.upper()))
errors = {}
if not galaxy_data['url']:
for k, v in galaxy_data.items():
if v:
setting_name = '{}{}'.format(prefix, k.upper())
errors.setdefault(setting_name, [])
errors[setting_name].append(_(
'Cannot provide field if PRIMARY_GALAXY_URL is not set.'
))
for k in GALAXY_SERVER_FIELDS:
if galaxy_data[k]:
setting_name = '{}{}'.format(prefix, k.upper())
if (not serializer.instance) or (not getattr(serializer.instance, setting_name, '')):
# new auth is applied, so check if compatible with version
from awx.main.utils import get_ansible_version
current_version = get_ansible_version()
min_version = '2.9'
if Version(current_version) < Version(min_version):
errors.setdefault(setting_name, [])
errors[setting_name].append(_(
'Galaxy server settings are not available until Ansible {min_version}, '
'you are running {current_version}.'
).format(min_version=min_version, current_version=current_version))
if (galaxy_data['password'] or galaxy_data['username']) and (galaxy_data['token'] or galaxy_data['auth_url']):
for k in ('password', 'username', 'token', 'auth_url'):
setting_name = '{}{}'.format(prefix, k.upper())
if setting_name in attrs:
errors.setdefault(setting_name, [])
errors[setting_name].append(_(
'Setting Galaxy token and authentication URL is mutually exclusive with username and password.'
))
if bool(galaxy_data['username']) != bool(galaxy_data['password']):
msg = _('If authenticating via username and password, both must be provided.')
for k in ('username', 'password'):
setting_name = '{}{}'.format(prefix, k.upper())
errors.setdefault(setting_name, [])
errors[setting_name].append(msg)
if bool(galaxy_data['token']) != bool(galaxy_data['auth_url']):
msg = _('If authenticating via token, both token and authentication URL must be provided.')
for k in ('token', 'auth_url'):
setting_name = '{}{}'.format(prefix, k.upper())
errors.setdefault(setting_name, [])
errors[setting_name].append(msg)
if errors:
raise serializers.ValidationError(errors)
return attrs
register_validate('logging', logging_validate)
register_validate('jobs', galaxy_validate)

4
awx/main/constants.py
View File

@ -51,3 +51,7 @@ LOGGER_BLACKLIST = (
# loggers that may be called getting logging settings
'awx.conf'
)
# these correspond to both AWX and Ansible settings to keep naming consistent
# for instance, settings.PRIMARY_GALAXY_AUTH_URL vs env var ANSIBLE_GALAXY_SERVER_FOO_AUTH_URL
GALAXY_SERVER_FIELDS = ('url', 'username', 'password', 'token', 'auth_url')

2
awx/main/models/credential/__init__.py
View File

@ -64,7 +64,7 @@ def build_safe_env(env):
for k, v in safe_env.items():
if k == 'AWS_ACCESS_KEY_ID':
continue
elif k.startswith('ANSIBLE_') and not k.startswith('ANSIBLE_NET'):
elif k.startswith('ANSIBLE_') and not k.startswith('ANSIBLE_NET') and not k.startswith('ANSIBLE_GALAXY_SERVER'):
continue
elif hidden_re.search(k):
safe_env[k] = HIDDEN_PASSWORD

12
awx/main/redact.py
View File

@ -1,6 +1,8 @@
import re
import urllib.parse as urlparse
from django.conf import settings
REPLACE_STR = '$encrypted$'
@ -10,14 +12,22 @@ class UriCleaner(object):
@staticmethod
def remove_sensitive(cleartext):
if settings.PRIMARY_GALAXY_URL:
exclude_list = [settings.PRIMARY_GALAXY_URL] + [server['url'] for server in settings.FALLBACK_GALAXY_SERVERS]
else:
exclude_list = [server['url'] for server in settings.FALLBACK_GALAXY_SERVERS]
redactedtext = cleartext
text_index = 0
while True:
match = UriCleaner.SENSITIVE_URI_PATTERN.search(redactedtext, text_index)
if not match:
break
uri_str = match.group(1)
# Do not redact items from the exclude list
if any(uri_str.startswith(exclude_uri) for exclude_uri in exclude_list):
text_index = match.start() + len(uri_str)
continue
try:
uri_str = match.group(1)
# May raise a ValueError if invalid URI for one reason or another
o = urlparse.urlsplit(uri_str)

20
awx/main/tasks.py
View File

@ -52,7 +52,7 @@ import ansible_runner
# AWX
from awx import __version__ as awx_application_version
from awx.main.constants import CLOUD_PROVIDERS, PRIVILEGE_ESCALATION_METHODS, STANDARD_INVENTORY_UPDATE_ENV
from awx.main.constants import CLOUD_PROVIDERS, PRIVILEGE_ESCALATION_METHODS, STANDARD_INVENTORY_UPDATE_ENV, GALAXY_SERVER_FIELDS
from awx.main.access import access_registry
from awx.main.models import (
Schedule, TowerScheduleState, Instance, InstanceGroup,
@ -1883,6 +1883,24 @@ class RunProjectUpdate(BaseTask):
env['TMP'] = settings.AWX_PROOT_BASE_PATH
env['PROJECT_UPDATE_ID'] = str(project_update.pk)
env['ANSIBLE_CALLBACK_PLUGINS'] = self.get_path_to('..', 'plugins', 'callback')
env['ANSIBLE_GALAXY_IGNORE'] = True
# Set up the fallback server, which is the normal Ansible Galaxy by default
galaxy_servers = list(settings.FALLBACK_GALAXY_SERVERS)
# If private galaxy URL is non-blank, that means this feature is enabled
if settings.PRIMARY_GALAXY_URL:
galaxy_servers = [{'id': 'primary_galaxy'}] + galaxy_servers
for key in GALAXY_SERVER_FIELDS:
value = getattr(settings, 'PRIMARY_GALAXY_{}'.format(key.upper()))
if value:
galaxy_servers[0][key] = value
for server in galaxy_servers:
for key in GALAXY_SERVER_FIELDS:
if not server.get(key):
continue
env_key = ('ANSIBLE_GALAXY_SERVER_{}_{}'.format(server.get('id', 'unnamed'), key)).upper()
env[env_key] = server[key]
# now set the precedence of galaxy servers
env['ANSIBLE_GALAXY_SERVER_LIST'] = ','.join([server.get('id', 'unnamed') for server in galaxy_servers])
return env
def _build_scm_url_extra_vars(self, project_update):

2
awx/main/tests/unit/test_tasks.py
View File

@ -130,6 +130,8 @@ def test_send_notifications_list(mock_notifications_filter, mock_job_get, mocker
('VMWARE_PASSWORD', 'SECRET'),
('API_SECRET', 'SECRET'),
('CALLBACK_CONNECTION', 'amqp://tower:password@localhost:5672/tower'),
('ANSIBLE_GALAXY_SERVER_PRIMARY_GALAXY_PASSWORD', 'SECRET'),
('ANSIBLE_GALAXY_SERVER_PRIMARY_GALAXY_TOKEN', 'SECRET'),
])
def test_safe_env_filtering(key, value):
assert build_safe_env({key: value})[key] == tasks.HIDDEN_PASSWORD

20
awx/settings/defaults.py
View File

@ -609,6 +609,9 @@ AWX_REBUILD_SMART_MEMBERSHIP = False
# By default, allow arbitrary Jinja templating in extra_vars defined on a Job Template
ALLOW_JINJA_IN_EXTRA_VARS = 'template'
# Run project updates with extra verbosity
PROJECT_UPDATE_VVV = False
# Enable dynamically pulling roles from a requirement.yml file
# when updating SCM projects
# Note: This setting may be overridden by database settings.
@ -619,6 +622,23 @@ AWX_ROLES_ENABLED = True
# Note: This setting may be overridden by database settings.
AWX_COLLECTIONS_ENABLED = True
# Settings for primary galaxy server, should be set in the UI
PRIMARY_GALAXY_URL = ''
PRIMARY_GALAXY_USERNAME = ''
PRIMARY_GALAXY_TOKEN = ''
PRIMARY_GALAXY_PASSWORD = ''
PRIMARY_GALAXY_AUTH_URL = ''
# Settings for the fallback galaxy server(s), normally this is the
# actual Ansible Galaxy site.
# server options: 'id', 'url', 'username', 'password', 'token', 'auth_url'
# To not use any fallback servers set this to []
FALLBACK_GALAXY_SERVERS = [
{
'id': 'galaxy',
'url': 'https://galaxy.ansible.com'
}
]
# Enable bubblewrap support for running jobs (playbook runs only).
# Note: This setting may be overridden by database settings.
AWX_PROOT_ENABLED = True

25
awx/ui/client/src/configuration/forms/jobs-form/configuration-jobs.form.js
View File

@ -58,12 +58,37 @@ export default ['i18n', function(i18n) {
type: 'text',
reset: 'ANSIBLE_FACT_CACHE_TIMEOUT',
},
PROJECT_UPDATE_VVV: {
type: 'toggleSwitch',
},
AWX_ROLES_ENABLED: {
type: 'toggleSwitch',
},
AWX_COLLECTIONS_ENABLED: {
type: 'toggleSwitch',
},
PRIMARY_GALAXY_URL: {
type: 'text',
reset: 'PRIMARY_GALAXY_URL',
},
PRIMARY_GALAXY_USERNAME: {
type: 'text',
reset: 'PRIMARY_GALAXY_USERNAME',
},
PRIMARY_GALAXY_PASSWORD: {
type: 'sensitive',
hasShowInputButton: true,
reset: 'PRIMARY_GALAXY_PASSWORD',
},
PRIMARY_GALAXY_TOKEN: {
type: 'sensitive',
hasShowInputButton: true,
reset: 'PRIMARY_GALAXY_TOKEN',
},
PRIMARY_GALAXY_AUTH_URL: {
type: 'text',
reset: 'PRIMARY_GALAXY_AUTH_URL',
},
AWX_TASK_ENV: {
type: 'textarea',
reset: 'AWX_TASK_ENV',

36
docs/collections.md
View File

@ -5,14 +5,14 @@ AWX supports the use of Ansible Collections. This section will give ways to use
### Project Collections Requirements
If you specify a Collections requirements file in SCM at `collections/requirements.yml`,
then AWX will install Collections in that file in the implicit project sync
before a job run. The invocation is:
then AWX will install Collections from that file in the implicit project sync
before a job run. The invocation looks like:
```
ansible-galaxy collection install -r requirements.yml -p <job tmp location>
ansible-galaxy collection install -r requirements.yml -p <job tmp location>/requirements_collections
```
Example of `tmp` directory where job is running:
Example of the resultant `tmp` directory where job is running:
```
├── project
@ -52,3 +52,31 @@ Example of `tmp` directory where job is running:
└── tmp_6wod58k
```
### Galaxy Server Selection
Ansible core default settings will download collections from the public
Galaxy server at `https://galaxy.ansible.com`. For details on
how Galaxy servers are configured in Ansible in general see:
https://docs.ansible.com/ansible/devel/user_guide/collections_using.html
(if "devel" link goes stale in the future, it is for Ansible 2.9)
You can set a different server to be the primary Galaxy server to download
roles and collections from in AWX project updates.
This is done via the setting `PRIMARY_GALAXY_URL` and similar
`PRIMARY_GALAXY_xxxx` settings for authentication.
If the `PRIMARY_GALAXY_URL` setting is not blank, then the server list is defined
to be `primary_galaxy,galaxy`. The `primary_galaxy` server definition uses the URL
from those settings, as well as username, password, and/or token and auth_url if applicable.
the `galaxy` server definition uses public Galaxy (`https://galaxy.ansible.com`)
with no authentication.
This configuration causes requirements to be downloaded from the user-specified
primary galaxy server if they are available there. If a requirement is
not available from the primary galaxy server, then it will fallback to
downloading it from the public Galaxy server.
Even when these settings are enabled, this can still be bypassed for a specific
requirement by using the `source:` option, as described in Ansible documentation.