From 296f70ce17b46dbfdd87ca6151b67001406c1746 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Thu, 21 Apr 2016 13:51:30 -0400 Subject: [PATCH 1/3] Implement Azure RM creds and inventory * Vendor ansible's azure_rm inventory script * Add new inventory type * Add new credential type * Expand host instance_id column from varchar 100 to 1024 to accept the long instance ids returned by Azure * Make the inventory_import azure match rename more explicit. --- awx/api/serializers.py | 2 +- awx/main/constants.py | 2 +- .../management/commands/inventory_import.py | 8 +- .../0019_v300_new_azure_credential.py | 55 ++ awx/main/models/base.py | 2 +- awx/main/models/credential.py | 29 +- awx/main/models/inventory.py | 9 +- awx/main/tasks.py | 33 +- awx/plugins/inventory/azure_rm.ini.example | 19 + awx/plugins/inventory/azure_rm.py | 786 ++++++++++++++++++ awx/settings/defaults.py | 10 + requirements/requirements_ansible.txt | 2 +- 12 files changed, 943 insertions(+), 14 deletions(-) create mode 100644 awx/main/migrations/0019_v300_new_azure_credential.py create mode 100644 awx/plugins/inventory/azure_rm.ini.example create mode 100755 awx/plugins/inventory/azure_rm.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 8884243a96..799f965acf 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1566,7 +1566,7 @@ class CredentialSerializer(BaseSerializer): 'password', 'security_token', 'project', 'domain', 'ssh_key_data', 'ssh_key_unlock', 'become_method', 'become_username', 'become_password', - 'vault_password') + 'vault_password', 'subscription', 'tenant', 'secret', 'client') def build_standard_field(self, field_name, model_field): field_class, field_kwargs = super(CredentialSerializer, self).build_standard_field(field_name, model_field) diff --git a/awx/main/constants.py b/awx/main/constants.py index ba65f7a715..6ecbc233ac 100644 --- a/awx/main/constants.py +++ b/awx/main/constants.py @@ -1,5 +1,5 @@ # Copyright (c) 2015 Ansible, Inc. # All Rights Reserved. -CLOUD_PROVIDERS = ('azure', 'ec2', 'gce', 'rax', 'vmware', 'openstack', 'foreman', 'cloudforms') +CLOUD_PROVIDERS = ('azure', 'azure_rm', 'ec2', 'gce', 'rax', 'vmware', 'openstack', 'foreman', 'cloudforms') SCHEDULEABLE_PROVIDERS = CLOUD_PROVIDERS + ('custom',) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 47d20cc1dd..9f350246bb 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -466,10 +466,10 @@ def load_inventory_source(source, all_group=None, group_filter_re=None, ''' Load inventory from given source directory or file. ''' - # Sanity check: We need the "azure" module to be titled "windows_azure.py", - # because it depends on the "azure" package from PyPI, and naming the - # module the same way makes the importer sad. - source = source.replace('azure', 'windows_azure') + # Sanity check: We sanitize these module names for our API but Ansible proper doesn't follow + # good naming conventions + if source == 'azure': + source = 'windows_azure' logger.debug('Analyzing type of source: %s', source) original_all_group = all_group diff --git a/awx/main/migrations/0019_v300_new_azure_credential.py b/awx/main/migrations/0019_v300_new_azure_credential.py new file mode 100644 index 0000000000..a787c9f38e --- /dev/null +++ b/awx/main/migrations/0019_v300_new_azure_credential.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0018_v300_host_ordering'), + ] + + operations = [ + migrations.AddField( + model_name='credential', + name='client', + field=models.CharField(default=b'', help_text='Client Id or Application Id for the credential', max_length=128, blank=True), + ), + migrations.AddField( + model_name='credential', + name='secret', + field=models.CharField(default=b'', help_text='Secret Token for this credential', max_length=1024, blank=True), + ), + migrations.AddField( + model_name='credential', + name='subscription', + field=models.CharField(default=b'', help_text='Subscription identifier for this credential', max_length=1024, blank=True), + ), + migrations.AddField( + model_name='credential', + name='tenant', + field=models.CharField(default=b'', help_text='Tenant identifier for this credential', max_length=1024, blank=True), + ), + migrations.AlterField( + model_name='credential', + name='kind', + field=models.CharField(default=b'ssh', max_length=32, choices=[(b'ssh', 'Machine'), (b'net', 'Network'), (b'scm', 'Source Control'), (b'aws', 'Amazon Web Services'), (b'rax', 'Rackspace'), (b'vmware', 'VMware vCenter'), (b'foreman', 'Satellite 6'), (b'cloudforms', 'CloudForms'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'openstack', 'OpenStack')]), + ), + migrations.AlterField( + model_name='host', + name='instance_id', + field=models.CharField(default=b'', max_length=1024, blank=True), + ), + migrations.AlterField( + model_name='inventorysource', + name='source', + field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'Local File, Directory or Script'), (b'rax', 'Rackspace Cloud Servers'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'foreman', 'Satellite 6'), (b'cloudforms', 'CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]), + ), + migrations.AlterField( + model_name='inventoryupdate', + name='source', + field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'Local File, Directory or Script'), (b'rax', 'Rackspace Cloud Servers'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'foreman', 'Satellite 6'), (b'cloudforms', 'CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]), + ), + + ] diff --git a/awx/main/models/base.py b/awx/main/models/base.py index 243d7612e8..4234f95d8f 100644 --- a/awx/main/models/base.py +++ b/awx/main/models/base.py @@ -61,7 +61,7 @@ PERMISSION_TYPE_CHOICES = [ (PERM_JOBTEMPLATE_CREATE, _('Create a Job Template')), ] -CLOUD_INVENTORY_SOURCES = ['ec2', 'rax', 'vmware', 'gce', 'azure', 'openstack', 'custom', 'foreman', 'cloudforms'] +CLOUD_INVENTORY_SOURCES = ['ec2', 'rax', 'vmware', 'gce', 'azure', 'azure_rm', 'openstack', 'custom', 'foreman', 'cloudforms'] VERBOSITY_CHOICES = [ (0, '0 (Normal)'), diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index bd24a72de5..0e880c7994 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -41,7 +41,8 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin): ('foreman', _('Satellite 6')), ('cloudforms', _('CloudForms')), ('gce', _('Google Compute Engine')), - ('azure', _('Microsoft Azure')), + ('azure', _('Microsoft Azure Classic (deprecated)')), + ('azure_rm', _('Microsoft Azure Resource Manager')), ('openstack', _('OpenStack')), ] @@ -55,7 +56,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin): ] PASSWORD_FIELDS = ('password', 'security_token', 'ssh_key_data', 'ssh_key_unlock', - 'become_password', 'vault_password') + 'become_password', 'vault_password', 'secret') class Meta: app_label = 'main' @@ -168,6 +169,30 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin): default='', help_text=_('Vault password (or "ASK" to prompt the user).'), ) + client = models.CharField( + max_length=128, + blank=True, + default='', + help_text=_('Client Id or Application Id for the credential'), + ) + secret = models.CharField( + max_length=1024, + blank=True, + default='', + help_text=_('Secret Token for this credential'), + ) + subscription = models.CharField( + max_length=1024, + blank=True, + default='', + help_text=_('Subscription identifier for this credential'), + ) + tenant = models.CharField( + max_length=1024, + blank=True, + default='', + help_text=_('Tenant identifier for this credential'), + ) owner_role = ImplicitRoleField( role_name='Credential Owner', role_description='Owner of the credential', diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index c42bb77967..ae66e4f48d 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -355,7 +355,7 @@ class Host(CommonModelNameNotUnique, ResourceMixin): help_text=_('Is this host online and available for running jobs?'), ) instance_id = models.CharField( - max_length=100, + max_length=1024, blank=True, default='', ) @@ -748,7 +748,8 @@ class InventorySourceOptions(BaseModel): ('rax', _('Rackspace Cloud Servers')), ('ec2', _('Amazon EC2')), ('gce', _('Google Compute Engine')), - ('azure', _('Microsoft Azure')), + ('azure', _('Microsoft Azure Classic (deprecated)')), + ('azure_rm', _('Microsoft Azure Resource Manager')), ('vmware', _('VMware vCenter')), ('foreman', _('Satellite 6')), ('cloudforms', _('CloudForms')), @@ -969,6 +970,10 @@ class InventorySourceOptions(BaseModel): regions.insert(0, ('all', 'All')) return regions + @classmethod + def get_azure_rm_region_choices(self): + return InventorySourceOptions.get_azure_region_choices() + @classmethod def get_vmware_region_choices(self): """Return a complete list of regions in VMware, as a list of two-tuples diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 2ec820f5c1..8c3d8a7826 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -801,6 +801,16 @@ class RunJob(BaseTask): elif cloud_cred and cloud_cred.kind == 'azure': env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.username env['AZURE_CERT_PATH'] = kwargs.get('private_data_files', {}).get('cloud_credential', '') + elif cloud_cred and cloud_cred.kind == 'azure_rm': + if len(cloud_cred.client) and len(cloud_cred.tenant): + env['AZURE_CLIENT_ID'] = cloud_cred.client + env['AZURE_SECRET'] = decrypt_field(cloud_cred, 'secret') + env['AZURE_TENANT'] = cloud_cred.tenant + env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.subscription + else: + env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.subscription + env['AZURE_AD_USER'] = cloud_cred.username + env['AZURE_PASSWORD'] = decrypt_field(cloud_cred, 'password') elif cloud_cred and cloud_cred.kind == 'vmware': env['VMWARE_USER'] = cloud_cred.username env['VMWARE_PASSWORD'] = decrypt_field(cloud_cred, 'password') @@ -1278,6 +1288,14 @@ class RunInventoryUpdate(BaseTask): cp.set(section, 'username', credential.username) cp.set(section, 'password', decrypt_field(credential, 'password')) + elif inventory_update.source == 'azure_rm': + section = 'azure' + cp.add_section(section) + cp.set(section, 'include_powerstate', 'yes') + cp.set(section, 'group_by_resource_group', 'yes') + cp.set(section, 'group_by_location', 'yes') + cp.set(section, 'group_by_tag', 'yes') + # Return INI content. if cp.sections(): f = cStringIO.StringIO() @@ -1298,9 +1316,9 @@ class RunInventoryUpdate(BaseTask): # passwords dictionary. credential = inventory_update.credential if credential: - for subkey in ('username', 'host', 'project'): + for subkey in ('username', 'host', 'project', 'client', 'tenant', 'subscription'): passwords['source_%s' % subkey] = getattr(credential, subkey) - for passkey in ('password', 'ssh_key_data', 'security_token'): + for passkey in ('password', 'ssh_key_data', 'security_token', 'secret'): k = 'source_%s' % passkey passwords[k] = decrypt_field(credential, passkey) return passwords @@ -1353,6 +1371,17 @@ class RunInventoryUpdate(BaseTask): elif inventory_update.source == 'azure': env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_username', '') env['AZURE_CERT_PATH'] = cloud_credential + elif inventory_update.source == 'azure_rm': + if len(passwords.get('source_client', '')) and \ + len(passwords.get('source_tenant', '')): + env['AZURE_CLIENT_ID'] = passwords.get('source_client', '') + env['AZURE_SECRET'] = passwords.get('source_secret', '') + env['AZURE_TENANT'] = passwords.get('source_tenant', '') + env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_subscription', '') + else: + env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_subscription', '') + env['AZURE_AD_USER'] = passwords.get('source_username', '') + env['AZURE_PASSWORD'] = passwords.get('source_password', '') elif inventory_update.source == 'gce': env['GCE_EMAIL'] = passwords.get('source_username', '') env['GCE_PROJECT'] = passwords.get('source_project', '') diff --git a/awx/plugins/inventory/azure_rm.ini.example b/awx/plugins/inventory/azure_rm.ini.example new file mode 100644 index 0000000000..6ea2688efa --- /dev/null +++ b/awx/plugins/inventory/azure_rm.ini.example @@ -0,0 +1,19 @@ +# +# Configuration file for azure_rm_invetory.py +# +[azure] +# Control which resource groups are included. By default all resources groups are included. +# Set resource_groups to a comma separated list of resource groups names. +#resource_groups= + +# Control which tags are included. Set tags to a comma separated list of keys or key:value pairs +#tags= + +# Include powerstate. If you don't need powerstate information, turning it off improves runtime performance. +include_powerstate=yes + +# Control grouping with the following boolean flags. Valid values: yes, no, true, false, True, False, 0, 1. +group_by_resource_group=yes +group_by_location=yes +group_by_security_group=no +group_by_tag=yes diff --git a/awx/plugins/inventory/azure_rm.py b/awx/plugins/inventory/azure_rm.py new file mode 100755 index 0000000000..0554ecf879 --- /dev/null +++ b/awx/plugins/inventory/azure_rm.py @@ -0,0 +1,786 @@ +#!/usr/bin/python +# +# Copyright (c) 2016 Matt Davis, +# Chris Houseknecht, +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +# + +''' +Azure External Inventory Script +=============================== +Generates dynamic inventory by making API requests to the Azure Resource +Manager using the AAzure Python SDK. For instruction on installing the +Azure Python SDK see http://azure-sdk-for-python.readthedocs.org/ + +Authentication +-------------- +The order of precedence is command line arguments, environment variables, +and finally the [default] profile found in ~/.azure/credentials. + +If using a credentials file, it should be an ini formatted file with one or +more sections, which we refer to as profiles. The script looks for a +[default] section, if a profile is not specified either on the command line +or with an environment variable. The keys in a profile will match the +list of command line arguments below. + +For command line arguments and environment variables specify a profile found +in your ~/.azure/credentials file, or a service principal or Active Directory +user. + +Command line arguments: + - profile + - client_id + - secret + - subscription_id + - tenant + - ad_user + - password + +Environment variables: + - AZURE_PROFILE + - AZURE_CLIENT_ID + - AZURE_SECRET + - AZURE_SUBSCRIPTION_ID + - AZURE_TENANT + - AZURE_AD_USER + - AZURE_PASSWORD + +Run for Specific Host +----------------------- +When run for a specific host using the --host option, a resource group is +required. For a specific host, this script returns the following variables: + +{ + "ansible_host": "XXX.XXX.XXX.XXX", + "computer_name": "computer_name2", + "fqdn": null, + "id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Compute/virtualMachines/object-name", + "image": { + "offer": "CentOS", + "publisher": "OpenLogic", + "sku": "7.1", + "version": "latest" + }, + "location": "westus", + "mac_address": "00-0D-3A-31-2C-EC", + "name": "object-name", + "network_interface": "interface-name", + "network_interface_id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Network/networkInterfaces/object-name1", + "network_security_group": null, + "network_security_group_id": null, + "os_disk": { + "name": "object-name", + "operating_system_type": "Linux" + }, + "plan": null, + "powerstate": "running", + "private_ip": "172.26.3.6", + "private_ip_alloc_method": "Static", + "provisioning_state": "Succeeded", + "public_ip": "XXX.XXX.XXX.XXX", + "public_ip_alloc_method": "Static", + "public_ip_id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Network/publicIPAddresses/object-name", + "public_ip_name": "object-name", + "resource_group": "galaxy-production", + "security_group": "object-name", + "security_group_id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Network/networkSecurityGroups/object-name", + "tags": { + "db": "database" + }, + "type": "Microsoft.Compute/virtualMachines", + "virtual_machine_size": "Standard_DS4" +} + +Groups +------ +When run in --list mode, instances are grouped by the following categories: + - azure + - location + - resource_group + - security_group + - tag key + - tag key_value + +Control groups using azure_rm_inventory.ini or set environment variables: + +AZURE_GROUP_BY_RESOURCE_GROUP=yes +AZURE_GROUP_BY_LOCATION=yes +AZURE_GROUP_BY_SECURITY_GROUP=yes +AZURE_GROUP_BY_TAG=yes + +Select hosts within specific resource groups by assigning a comma separated list to: + +AZURE_RESOURCE_GROUPS=resource_group_a,resource_group_b + +Select hosts for specific tag key by assigning a comma separated list of tag keys to: + +AZURE_TAGS=key1,key2,key3 + +Or, select hosts for specific tag key:value pairs by assigning a comma separated list key:value pairs to: + +AZURE_TAGS=key1:value1,key2:value2 + +If you don't need the powerstate, you can improve performance by turning off powerstate fetching: +AZURE_INCLUDE_POWERSTATE=no + +azure_rm_inventory.ini +---------------------- +As mentioned above you can control execution using environment variables or an .ini file. A sample +azure_rm_inventory.ini is included. The name of the .ini file is the basename of the inventory script (in this case +'azure_rm_inventory') with a .ini extension. This provides you with the flexibility of copying and customizing this +script and having matching .ini files. Go forth and customize your Azure inventory! + +Powerstate: +----------- +The powerstate attribute indicates whether or not a host is running. If the value is 'running', the machine is +up. If the value is anything other than 'running', the machine is down, and will be unreachable. + +Examples: +--------- + Execute /bin/uname on all instances in the galaxy-qa resource group + $ ansible -i azure_rm_inventory.py galaxy-qa -m shell -a "/bin/uname -a" + + Use the inventory script to print instance specific information + $ contrib/inventory/azure_rm_inventory.py --host my_instance_host_name --pretty + + Use with a playbook + $ ansible-playbook -i contrib/inventory/azure_rm_inventory.py my_playbook.yml --limit galaxy-qa + + +Insecure Platform Warning +------------------------- +If you receive InsecurePlatformWarning from urllib3, install the +requests security packages: + + pip install requests[security] + + +author: + - Chris Houseknecht (@chouseknecht) + - Matt Davis (@nitzmahone) + +Company: Ansible by Red Hat + +Version: 1.0.0 +''' + +import argparse +import ConfigParser +import json +import os +import re +import sys + +from os.path import expanduser + +HAS_AZURE = True +HAS_AZURE_EXC = None + +try: + from msrestazure.azure_exceptions import CloudError + from azure.mgmt.compute import __version__ as azure_compute_version + from azure.common import AzureMissingResourceHttpError, AzureHttpError + from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials + from azure.mgmt.network.network_management_client import NetworkManagementClient,\ + NetworkManagementClientConfiguration + from azure.mgmt.resource.resources.resource_management_client import ResourceManagementClient,\ + ResourceManagementClientConfiguration + from azure.mgmt.compute.compute_management_client import ComputeManagementClient,\ + ComputeManagementClientConfiguration +except ImportError as exc: + HAS_AZURE_EXC = exc + HAS_AZURE = False + + +AZURE_CREDENTIAL_ENV_MAPPING = dict( + profile='AZURE_PROFILE', + subscription_id='AZURE_SUBSCRIPTION_ID', + client_id='AZURE_CLIENT_ID', + secret='AZURE_SECRET', + tenant='AZURE_TENANT', + ad_user='AZURE_AD_USER', + password='AZURE_PASSWORD' +) + +AZURE_CONFIG_SETTINGS = dict( + resource_groups='AZURE_RESOURCE_GROUPS', + tags='AZURE_TAGS', + include_powerstate='AZURE_INCLUDE_POWERSTATE', + group_by_resource_group='AZURE_GROUP_BY_RESOURCE_GROUP', + group_by_location='AZURE_GROUP_BY_LOCATION', + group_by_security_group='AZURE_GROUP_BY_SECURITY_GROUP', + group_by_tag='AZURE_GROUP_BY_TAG' +) + +AZURE_MIN_VERSION = "2016-03-30" + + +def azure_id_to_dict(id): + pieces = re.sub(r'^\/', '', id).split('/') + result = {} + index = 0 + while index < len(pieces) - 1: + result[pieces[index]] = pieces[index + 1] + index += 1 + return result + + +class AzureRM(object): + + def __init__(self, args): + self._args = args + self._compute_client = None + self._resource_client = None + self._network_client = None + + self.debug = False + if args.debug: + self.debug = True + + self.credentials = self._get_credentials(args) + if not self.credentials: + self.fail("Failed to get credentials. Either pass as parameters, set environment variables, " + "or define a profile in ~/.azure/credentials.") + + if self.credentials.get('subscription_id', None) is None: + self.fail("Credentials did not include a subscription_id value.") + self.log("setting subscription_id") + self.subscription_id = self.credentials['subscription_id'] + + if self.credentials.get('client_id') is not None and \ + self.credentials.get('secret') is not None and \ + self.credentials.get('tenant') is not None: + self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'], + secret=self.credentials['secret'], + tenant=self.credentials['tenant']) + elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None: + self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password']) + else: + self.fail("Failed to authenticate with provided credentials. Some attributes were missing. " + "Credentials must include client_id, secret and tenant or ad_user and password.") + + def log(self, msg): + if self.debug: + print (msg + u'\n') + + def fail(self, msg): + raise Exception(msg) + + def _get_profile(self, profile="default"): + path = expanduser("~") + path += "/.azure/credentials" + try: + config = ConfigParser.ConfigParser() + config.read(path) + except Exception as exc: + self.fail("Failed to access {0}. Check that the file exists and you have read " + "access. {1}".format(path, str(exc))) + credentials = dict() + for key in AZURE_CREDENTIAL_ENV_MAPPING: + try: + credentials[key] = config.get(profile, key, raw=True) + except: + pass + + if credentials.get('client_id') is not None or credentials.get('ad_user') is not None: + return credentials + + return None + + def _get_env_credentials(self): + env_credentials = dict() + for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.iteritems(): + env_credentials[attribute] = os.environ.get(env_variable, None) + + if env_credentials['profile'] is not None: + credentials = self._get_profile(env_credentials['profile']) + return credentials + + if env_credentials['client_id'] is not None or env_credentials['ad_user'] is not None: + return env_credentials + + return None + + def _get_credentials(self, params): + # Get authentication credentials. + # Precedence: cmd line parameters-> environment variables-> default profile in ~/.azure/credentials. + + self.log('Getting credentials') + + arg_credentials = dict() + for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.iteritems(): + arg_credentials[attribute] = getattr(params, attribute) + + # try module params + if arg_credentials['profile'] is not None: + self.log('Retrieving credentials with profile parameter.') + credentials = self._get_profile(arg_credentials['profile']) + return credentials + + if arg_credentials['client_id'] is not None: + self.log('Received credentials from parameters.') + return arg_credentials + + # try environment + env_credentials = self._get_env_credentials() + if env_credentials: + self.log('Received credentials from env.') + return env_credentials + + # try default profile from ~./azure/credentials + default_credentials = self._get_profile() + if default_credentials: + self.log('Retrieved default profile credentials from ~/.azure/credentials.') + return default_credentials + + return None + + def _register(self, key): + try: + # We have to perform the one-time registration here. Otherwise, we receive an error the first + # time we attempt to use the requested client. + resource_client = self.rm_client + resource_client.providers.register(key) + except Exception as exc: + self.fail("One-time registration of {0} failed - {1}".format(key, str(exc))) + + @property + def network_client(self): + self.log('Getting network client') + if not self._network_client: + self._network_client = NetworkManagementClient( + NetworkManagementClientConfiguration(self.azure_credentials, self.subscription_id)) + self._register('Microsoft.Network') + return self._network_client + + @property + def rm_client(self): + self.log('Getting resource manager client') + if not self._resource_client: + self._resource_client = ResourceManagementClient( + ResourceManagementClientConfiguration(self.azure_credentials, self.subscription_id)) + return self._resource_client + + @property + def compute_client(self): + self.log('Getting compute client') + if not self._compute_client: + self._compute_client = ComputeManagementClient( + ComputeManagementClientConfiguration(self.azure_credentials, self.subscription_id)) + self._register('Microsoft.Compute') + return self._compute_client + + +class AzureInventory(object): + + def __init__(self): + + self._args = self._parse_cli_args() + + try: + rm = AzureRM(self._args) + except Exception as e: + sys.exit("{0}".format(str(e))) + + self._compute_client = rm.compute_client + self._network_client = rm.network_client + self._resource_client = rm.rm_client + self._security_groups = None + + self.resource_groups = [] + self.tags = None + self.replace_dash_in_groups = False + self.group_by_resource_group = True + self.group_by_location = True + self.group_by_security_group = True + self.group_by_tag = True + self.include_powerstate = True + + self._inventory = dict( + _meta=dict( + hostvars=dict() + ), + azure=[] + ) + + self._get_settings() + + if self._args.resource_groups: + self.resource_groups = self._args.resource_groups.split(',') + + if self._args.tags: + self.tags = self._args.tags.split(',') + + if self._args.no_powerstate: + self.include_powerstate = False + + self.get_inventory() + print (self._json_format_dict(pretty=self._args.pretty)) + sys.exit(0) + + def _parse_cli_args(self): + # Parse command line arguments + parser = argparse.ArgumentParser( + description='Produce an Ansible Inventory file for an Azure subscription') + parser.add_argument('--list', action='store_true', default=True, + help='List instances (default: True)') + parser.add_argument('--debug', action='store_true', default=False, + help='Send debug messages to STDOUT') + parser.add_argument('--host', action='store', + help='Get all information about an instance') + parser.add_argument('--pretty', action='store_true', default=False, + help='Pretty print JSON output(default: False)') + parser.add_argument('--profile', action='store', + help='Azure profile contained in ~/.azure/credentials') + parser.add_argument('--subscription_id', action='store', + help='Azure Subscription Id') + parser.add_argument('--client_id', action='store', + help='Azure Client Id ') + parser.add_argument('--secret', action='store', + help='Azure Client Secret') + parser.add_argument('--tenant', action='store', + help='Azure Tenant Id') + parser.add_argument('--ad-user', action='store', + help='Active Directory User') + parser.add_argument('--password', action='store', + help='password') + parser.add_argument('--resource-groups', action='store', + help='Return inventory for comma separated list of resource group names') + parser.add_argument('--tags', action='store', + help='Return inventory for comma separated list of tag key:value pairs') + parser.add_argument('--no-powerstate', action='store_true', default=False, + help='Do not include the power state of each virtual host') + return parser.parse_args() + + def get_inventory(self): + if len(self.resource_groups) > 0: + # get VMs for requested resource groups + for resource_group in self.resource_groups: + try: + virtual_machines = self._compute_client.virtual_machines.list(resource_group) + except Exception as exc: + sys.exit("Error: fetching virtual machines for resource group {0} - {1}".format(resource_group, + str(exc))) + if self._args.host or self.tags: + selected_machines = self._selected_machines(virtual_machines) + self._load_machines(selected_machines) + else: + self._load_machines(virtual_machines) + else: + # get all VMs within the subscription + try: + virtual_machines = self._compute_client.virtual_machines.list_all() + except Exception as exc: + sys.exit("Error: fetching virtual machines - {0}".format(str(exc))) + + if self._args.host or self.tags > 0: + selected_machines = self._selected_machines(virtual_machines) + self._load_machines(selected_machines) + else: + self._load_machines(virtual_machines) + + def _load_machines(self, machines): + for machine in machines: + id_dict = azure_id_to_dict(machine.id) + + #TODO - The API is returning an ID value containing resource group name in ALL CAPS. If/when it gets + # fixed, we should remove the .lower(). Opened Issue + # #574: https://github.com/Azure/azure-sdk-for-python/issues/574 + resource_group = id_dict['resourceGroups'].lower() + + if self.group_by_security_group: + self._get_security_groups(resource_group) + + host_vars = dict( + ansible_host=None, + private_ip=None, + private_ip_alloc_method=None, + public_ip=None, + public_ip_name=None, + public_ip_id=None, + public_ip_alloc_method=None, + fqdn=None, + location=machine.location, + name=machine.name, + type=machine.type, + id=machine.id, + tags=machine.tags, + network_interface_id=None, + network_interface=None, + resource_group=resource_group, + mac_address=None, + plan=(machine.plan.name if machine.plan else None), + virtual_machine_size=machine.hardware_profile.vm_size.value, + computer_name=machine.os_profile.computer_name, + provisioning_state=machine.provisioning_state, + ) + + host_vars['os_disk'] = dict( + name=machine.storage_profile.os_disk.name, + operating_system_type=machine.storage_profile.os_disk.os_type.value + ) + + if self.include_powerstate: + host_vars['powerstate'] = self._get_powerstate(resource_group, machine.name) + + if machine.storage_profile.image_reference: + host_vars['image'] = dict( + offer=machine.storage_profile.image_reference.offer, + publisher=machine.storage_profile.image_reference.publisher, + sku=machine.storage_profile.image_reference.sku, + version=machine.storage_profile.image_reference.version + ) + + # Add windows details + if machine.os_profile.windows_configuration is not None: + host_vars['windows_auto_updates_enabled'] = \ + machine.os_profile.windows_configuration.enable_automatic_updates + host_vars['windows_timezone'] = machine.os_profile.windows_configuration.time_zone + host_vars['windows_rm'] = None + if machine.os_profile.windows_configuration.win_rm is not None: + host_vars['windows_rm'] = dict(listeners=None) + if machine.os_profile.windows_configuration.win_rm.listeners is not None: + host_vars['windows_rm']['listeners'] = [] + for listener in machine.os_profile.windows_configuration.win_rm.listeners: + host_vars['windows_rm']['listeners'].append(dict(protocol=listener.protocol, + certificate_url=listener.certificate_url)) + + for interface in machine.network_profile.network_interfaces: + interface_reference = self._parse_ref_id(interface.id) + network_interface = self._network_client.network_interfaces.get( + interface_reference['resourceGroups'], + interface_reference['networkInterfaces']) + if network_interface.primary: + if self.group_by_security_group and \ + self._security_groups[resource_group].get(network_interface.id, None): + host_vars['security_group'] = \ + self._security_groups[resource_group][network_interface.id]['name'] + host_vars['security_group_id'] = \ + self._security_groups[resource_group][network_interface.id]['id'] + host_vars['network_interface'] = network_interface.name + host_vars['network_interface_id'] = network_interface.id + host_vars['mac_address'] = network_interface.mac_address + for ip_config in network_interface.ip_configurations: + host_vars['private_ip'] = ip_config.private_ip_address + host_vars['private_ip_alloc_method'] = ip_config.private_ip_allocation_method.value + if ip_config.public_ip_address: + public_ip_reference = self._parse_ref_id(ip_config.public_ip_address.id) + public_ip_address = self._network_client.public_ip_addresses.get( + public_ip_reference['resourceGroups'], + public_ip_reference['publicIPAddresses']) + host_vars['ansible_host'] = public_ip_address.ip_address + host_vars['public_ip'] = public_ip_address.ip_address + host_vars['public_ip_name'] = public_ip_address.name + host_vars['public_ip_alloc_method'] = public_ip_address.public_ip_allocation_method.value + host_vars['public_ip_id'] = public_ip_address.id + if public_ip_address.dns_settings: + host_vars['fqdn'] = public_ip_address.dns_settings.fqdn + + self._add_host(host_vars) + + def _selected_machines(self, virtual_machines): + selected_machines = [] + for machine in virtual_machines: + if self._args.host and self._args.host == machine.name: + selected_machines.append(machine) + if self.tags and self._tags_match(machine.tags, self.tags): + selected_machines.append(machine) + return selected_machines + + def _get_security_groups(self, resource_group): + ''' For a given resource_group build a mapping of network_interface.id to security_group name ''' + if not self._security_groups: + self._security_groups = dict() + if not self._security_groups.get(resource_group): + self._security_groups[resource_group] = dict() + for group in self._network_client.network_security_groups.list(resource_group): + if group.network_interfaces: + for interface in group.network_interfaces: + self._security_groups[resource_group][interface.id] = dict( + name=group.name, + id=group.id + ) + + def _get_powerstate(self, resource_group, name): + try: + vm = self._compute_client.virtual_machines.get(resource_group, + name, + expand='instanceview') + except Exception as exc: + sys.exit("Error: fetching instanceview for host {0} - {1}".format(name, str(exc))) + + return next((s.code.replace('PowerState/', '') + for s in vm.instance_view.statuses if s.code.startswith('PowerState')), None) + + def _add_host(self, vars): + + host_name = self._to_safe(vars['name']) + resource_group = self._to_safe(vars['resource_group']) + security_group = None + if vars.get('security_group'): + security_group = self._to_safe(vars['security_group']) + + if self.group_by_resource_group: + if not self._inventory.get(resource_group): + self._inventory[resource_group] = [] + self._inventory[resource_group].append(host_name) + + if self.group_by_location: + if not self._inventory.get(vars['location']): + self._inventory[vars['location']] = [] + self._inventory[vars['location']].append(host_name) + + if self.group_by_security_group and security_group: + if not self._inventory.get(security_group): + self._inventory[security_group] = [] + self._inventory[security_group].append(host_name) + + self._inventory['_meta']['hostvars'][host_name] = vars + self._inventory['azure'].append(host_name) + + if self.group_by_tag and vars.get('tags'): + for key, value in vars['tags'].iteritems(): + safe_key = self._to_safe(key) + safe_value = safe_key + '_' + self._to_safe(value) + if not self._inventory.get(safe_key): + self._inventory[safe_key] = [] + if not self._inventory.get(safe_value): + self._inventory[safe_value] = [] + self._inventory[safe_key].append(host_name) + self._inventory[safe_value].append(host_name) + + def _json_format_dict(self, pretty=False): + # convert inventory to json + if pretty: + return json.dumps(self._inventory, sort_keys=True, indent=2) + else: + return json.dumps(self._inventory) + + def _get_settings(self): + # Load settings from the .ini, if it exists. Otherwise, + # look for environment values. + file_settings = self._load_settings() + if file_settings: + for key in AZURE_CONFIG_SETTINGS: + if key in ('resource_groups', 'tags') and file_settings.get(key, None) is not None: + values = file_settings.get(key).split(',') + if len(values) > 0: + setattr(self, key, values) + elif file_settings.get(key, None) is not None: + val = self._to_boolean(file_settings[key]) + setattr(self, key, val) + else: + env_settings = self._get_env_settings() + for key in AZURE_CONFIG_SETTINGS: + if key in('resource_groups', 'tags') and env_settings.get(key, None) is not None: + values = env_settings.get(key).split(',') + if len(values) > 0: + setattr(self, key, values) + elif env_settings.get(key, None) is not None: + val = self._to_boolean(env_settings[key]) + setattr(self, key, val) + + def _parse_ref_id(self, reference): + response = {} + keys = reference.strip('/').split('/') + for index in range(len(keys)): + if index < len(keys) - 1 and index % 2 == 0: + response[keys[index]] = keys[index + 1] + return response + + def _to_boolean(self, value): + if value in ['Yes', 'yes', 1, 'True', 'true', True]: + result = True + elif value in ['No', 'no', 0, 'False', 'false', False]: + result = False + else: + result = True + return result + + def _get_env_settings(self): + env_settings = dict() + for attribute, env_variable in AZURE_CONFIG_SETTINGS.iteritems(): + env_settings[attribute] = os.environ.get(env_variable, None) + return env_settings + + def _load_settings(self): + basename = os.path.splitext(os.path.basename(__file__))[0] + path = basename + '.ini' + config = None + settings = None + try: + config = ConfigParser.ConfigParser() + config.read(path) + except: + pass + + if config is not None: + settings = dict() + for key in AZURE_CONFIG_SETTINGS: + try: + settings[key] = config.get('azure', key, raw=True) + except: + pass + + return settings + + def _tags_match(self, tag_obj, tag_args): + ''' + Return True if the tags object from a VM contains the requested tag values. + + :param tag_obj: Dictionary of string:string pairs + :param tag_args: List of strings in the form key=value + :return: boolean + ''' + + if not tag_obj: + return False + + matches = 0 + for arg in tag_args: + arg_key = arg + arg_value = None + if re.search(r':', arg): + arg_key, arg_value = arg.split(':') + if arg_value and tag_obj.get(arg_key, None) == arg_value: + matches += 1 + elif not arg_value and tag_obj.get(arg_key, None) is not None: + matches += 1 + if matches == len(tag_args): + return True + return False + + def _to_safe(self, word): + ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups ''' + regex = "[^A-Za-z0-9\_" + if not self.replace_dash_in_groups: + regex += "\-" + return re.sub(regex + "]", "_", word) + + +def main(): + if not HAS_AZURE: + sys.exit("The Azure python sdk is not installed (try 'pip install azure') - {0}".format(HAS_AZURE_EXC)) + + if azure_compute_version < AZURE_MIN_VERSION: + sys.exit("Expecting azure.mgmt.compute.__version__ to be >= {0}. Found version {1} " + "Do you have Azure >= 2.0.0rc2 installed?".format(AZURE_MIN_VERSION, azure_compute_version)) + + AzureInventory() + +if __name__ == '__main__': + main() diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 3d189f76ef..8546f49878 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -651,6 +651,16 @@ AZURE_HOST_FILTER = r'^.+$' AZURE_EXCLUDE_EMPTY_GROUPS = True AZURE_INSTANCE_ID_VAR = 'private_id' +# -------------------------------------- +# -- Microsoft Azure Resource Manager -- +# -------------------------------------- +AZURE_RM_GROUP_FILTER = r'^.+$' +AZURE_RM_HOST_FILTER = r'^.+$' +AZURE_RM_ENABLED_VAR = 'powerstate' +AZURE_RM_ENABLED_VALUE = 'running' +AZURE_RM_INSTANCE_ID_VAR = 'id' +AZURE_RM_EXCLUDE_EMPTY_GROUPS = True + # --------------------- # ----- OpenStack ----- # --------------------- diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt index 58b10f7fef..bf318d2355 100644 --- a/requirements/requirements_ansible.txt +++ b/requirements/requirements_ansible.txt @@ -1,7 +1,7 @@ anyjson==0.3.3 apache-libcloud==0.15.1 appdirs==1.4.0 -azure==0.9.0 +azure==2.0.0rc2 Babel==2.2.0 boto==2.34.0 cliff==1.15.0 From 0cd4b52582dccaf9bd1b91fcc2e87f87326d0f9d Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Thu, 21 Apr 2016 14:03:27 -0400 Subject: [PATCH 2/3] Remove azure dependency from tower venv --- requirements/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 6111ac92ae..3e0bea4550 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -3,7 +3,6 @@ amqp==1.4.5 anyjson==0.3.3 apache-libcloud==0.15.1 appdirs==1.4.0 -azure==0.9.0 Babel==2.2.0 billiard==3.3.0.16 boto==2.34.0 From e1808a6ed7f5b52f8854d09c0f642ef81dc3efda Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Thu, 21 Apr 2016 14:13:03 -0400 Subject: [PATCH 3/3] Update indentation for flake8 --- awx/main/tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 8c3d8a7826..4d08944786 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1374,10 +1374,10 @@ class RunInventoryUpdate(BaseTask): elif inventory_update.source == 'azure_rm': if len(passwords.get('source_client', '')) and \ len(passwords.get('source_tenant', '')): - env['AZURE_CLIENT_ID'] = passwords.get('source_client', '') - env['AZURE_SECRET'] = passwords.get('source_secret', '') - env['AZURE_TENANT'] = passwords.get('source_tenant', '') - env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_subscription', '') + env['AZURE_CLIENT_ID'] = passwords.get('source_client', '') + env['AZURE_SECRET'] = passwords.get('source_secret', '') + env['AZURE_TENANT'] = passwords.get('source_tenant', '') + env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_subscription', '') else: env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_subscription', '') env['AZURE_AD_USER'] = passwords.get('source_username', '')