shaba/awx
1
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2024-10-31 15:21:13 +03:00
Code Releases Activity

Fixes https://trello.com/c/8KGzMa75 - Updates to VMware inventory script to include additional host variables, groups and ansible_ssh_host.

Browse Source
This commit is contained in:
Chris Church 2014-10-30 13:47:38 -04:00
parent 202b511f55
commit f5a174f991
5 changed files with 417 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
awx/main/tasks.py
View File

@ -956,7 +956,7 @@ class RunInventoryUpdate(BaseTask):
ec2_opts['cache_path'] = cache_path
ec2_opts.setdefault('cache_max_age', '300')
for k,v in ec2_opts.items():
cp.set(section, k, str(v))
cp.set(section, k, unicode(v))
# Build pyrax creds INI for rax inventory script.
elif inventory_update.source == 'rax':
section = 'rackspace_cloud'
@ -966,6 +966,15 @@ class RunInventoryUpdate(BaseTask):
cp.set(section, 'username', credential.username)
cp.set(section, 'api_key', decrypt_field(credential,
'password'))
# Allow custom options to vmware inventory script.
elif inventory_update.source == 'vmware':
section = 'defaults'
cp.add_section(section)
vmware_opts = dict(inventory_update.source_vars_dict.items())
vmware_opts.setdefault('guests_only', 'True')
for k,v in vmware_opts.items():
cp.set(section, k, unicode(v))
# Return INI content.
if cp.sections():
f = cStringIO.StringIO()
@ -1026,6 +1035,7 @@ class RunInventoryUpdate(BaseTask):
# complain about not being able to determine its version number.
env['PBR_VERSION'] = '0.5.21'
elif inventory_update.source == 'vmware':
env['VMWARE_INI'] = kwargs.get('private_data_file', '')
env['VMWARE_HOST'] = passwords.get('source_host', '')
env['VMWARE_USER'] = passwords.get('source_username', '')
env['VMWARE_PASSWORD'] = passwords.get('source_password', '')

56
awx/main/tests/inventory.py
View File

@ -1562,6 +1562,62 @@ class InventoryUpdatesTest(BaseTransactionTest):
# its own child).
self.assertTrue(self.group in self.inventory.root_groups)
def test_update_from_vmware(self):
source_host = getattr(settings, 'TEST_VMWARE_HOST', '')
source_username = getattr(settings, 'TEST_VMWARE_USER', '')
source_password = getattr(settings, 'TEST_VMWARE_PASSWORD', '')
if not all([source_host, source_username, source_password]):
self.skipTest('no test vmware credentials defined!')
self.create_test_license_file()
credential = Credential.objects.create(kind='vmware',
user=self.super_django_user,
username=source_username,
password=source_password,
host=source_host)
inventory_source = self.update_inventory_source(self.group,
source='vmware', credential=credential)
# Check first without instance_id set (to import by name only).
with self.settings(VMWARE_INSTANCE_ID_VAR=''):
self.check_inventory_source(inventory_source)
# Rename hosts and verify the import picks up the instance_id present
# in host variables.
for host in self.inventory.hosts.all():
self.assertFalse(host.instance_id, host.instance_id)
if host.enabled:
self.assertTrue(host.variables_dict.get('ansible_ssh_host', ''))
# Test a field that should be present for host systems, not VMs.
self.assertFalse(host.variables_dict.get('vmware_product_name', ''))
host.name = 'updated-%s' % host.name
host.save()
old_host_pks = set(self.inventory.hosts.values_list('pk', flat=True))
self.check_inventory_source(inventory_source, initial=False)
new_host_pks = set(self.inventory.hosts.values_list('pk', flat=True))
self.assertEqual(old_host_pks, new_host_pks)
# Manually disable all hosts, verify a new update re-enables them.
# Also change the host name, and verify it is not deleted, but instead
# updated because the instance ID matches.
enabled_host_pks = set(self.inventory.hosts.filter(enabled=True).values_list('pk', flat=True))
for host in self.inventory.hosts.all():
host.enabled = False
host.name = 'changed-%s' % host.name
host.save()
old_host_pks = set(self.inventory.hosts.values_list('pk', flat=True))
self.check_inventory_source(inventory_source, initial=False, enabled_host_pks=enabled_host_pks)
new_host_pks = set(self.inventory.hosts.values_list('pk', flat=True))
self.assertEqual(old_host_pks, new_host_pks)
# Update again and include host systems in addition to guests.
inventory_source.source_vars = '---\n\nguests_only: false\n'
inventory_source.save()
old_host_pks = set(self.inventory.hosts.values_list('pk', flat=True))
self.check_inventory_source(inventory_source, initial=False)
new_host_pks = set(self.inventory.hosts.values_list('pk', flat=True))
self.assertTrue(new_host_pks > old_host_pks)
for host in self.inventory.hosts.filter(pk__in=(new_host_pks - old_host_pks)):
if host.enabled:
self.assertTrue(host.variables_dict.get('ansible_ssh_host', ''))
# Test a field only present for host systems.
self.assertTrue(host.variables_dict.get('vmware_product_name', ''))
def test_update_from_custom_script(self):
# Create the inventory script
self.create_test_license_file()

512
awx/plugins/inventory/vmware.py
View File

@ -1,8 +1,8 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
VMWARE external inventory script
=================================
VMware Inventory Script
=======================
shamelessly copied from existing inventory scripts.
@ -14,219 +14,391 @@ i.e vmware.py/vmware_colo.ini vmware_idf.py/vmware_idf.ini
so if you don't have clustered vcenter but multiple esx machines or
just diff clusters you can have a inventory per each and automatically
group hosts based on file name or specify a group in the ini.
FIXME
'''
import collections
import json
import logging
import optparse
import os
import sys
import time
import ConfigParser
from psphere.client import Client
from psphere.managedobjects import HostSystem
# Disable logging message trigged by pSphere/suds.
try:
import json
from logging import NullHandler
except ImportError:
import simplejson as json
from logging import Handler
class NullHandler(Handler):
def emit(self, record):
pass
logging.getLogger('psphere').addHandler(NullHandler())
logging.getLogger('suds').addHandler(NullHandler())
from psphere.client import Client
from psphere.errors import ObjectNotFoundError
from psphere.managedobjects import HostSystem, VirtualMachine, ManagedObject, Network
from suds.sudsobject import Object as SudsObject
def save_cache(cache_item, data, config):
''' saves item to cache '''
class VMwareInventory(object):
def __init__(self, guests_only=None):
self.config = ConfigParser.SafeConfigParser()
if os.environ.get('VMWARE_INI', ''):
config_files = [os.environ['VMWARE_INI']]
else:
config_files = [os.path.abspath(sys.argv[0]).rstrip('.py') + '.ini', 'vmware.ini']
for config_file in config_files:
if os.path.exists(config_file):
self.config.read(config_file)
break
# Sanity check: Is caching enabled? If not, don't cache.
if not config.has_option('defaults', 'cache_dir'):
return
# Retrieve only guest VMs, or include host systems?
if guests_only is not None:
self.guests_only = guests_only
elif self.config.has_option('defaults', 'guests_only'):
self.guests_only = self.config.getboolean('defaults', 'guests_only')
else:
self.guests_only = True
dpath = config.get('defaults', 'cache_dir')
try:
cache = open('/'.join([dpath,cache_item]), 'w')
cache.write(json.dumps(data))
cache.close()
except IOError, e:
pass # not really sure what to do here
# Read authentication information from VMware environment variables
# (if set), otherwise from INI file.
auth_host = os.environ.get('VMWARE_HOST')
if not auth_host and self.config.has_option('auth', 'host'):
auth_host = self.config.get('auth', 'host')
auth_user = os.environ.get('VMWARE_USER')
if not auth_user and self.config.has_option('auth', 'user'):
auth_user = self.config.get('auth', 'user')
auth_password = os.environ.get('VMWARE_PASSWORD')
if not auth_password and self.config.has_option('auth', 'password'):
auth_password = self.config.get('auth', 'password')
# Create the VMware client connection.
self.client = Client(auth_host, auth_user, auth_password)
def get_cache(cache_item, config):
''' returns cached item '''
def _put_cache(self, name, value):
'''
Saves the value to cache with the name given.
'''
if self.config.has_option('defaults', 'cache_dir'):
cache_dir = self.config.get('defaults', 'cache_dir')
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
cache_file = os.path.join(cache_dir, name)
with open(cache_file, 'w') as cache:
json.dump(value, cache)
# Sanity check: Is caching enabled? If not, return None.
if not config.has_option('defaults', 'cache_dir'):
return
def _get_cache(self, name, default=None):
'''
Retrieves the value from cache for the given name.
'''
if self.config.has_option('defaults', 'cache_dir'):
cache_dir = self.config.get('defaults', 'cache_dir')
cache_file = os.path.join(cache_dir, name)
if os.path.exists(cache_file):
if self.config.has_option('defaults', 'cache_max_age'):
cache_max_age = self.config.getint('defaults', 'cache_max_age')
else:
cache_max_age = 0
cache_stat = os.stat(cache_file)
if (cache_stat.st_mtime + cache_max_age) < time.time():
with open(cache_file) as cache:
return json.load(cache)
return default
dpath = config.get('defaults', 'cache_dir')
inv = {}
try:
cache = open('/'.join([dpath,cache_item]), 'r')
inv = json.loads(cache.read())
cache.close()
except IOError, e:
pass # not really sure what to do here
def _flatten_dict(self, d, parent_key='', sep='_'):
'''
Flatten nested dicts by combining keys with a separator. Lists with
only string items are included as is; any other lists are discarded.
'''
items = []
for k, v in d.items():
if k.startswith('_'):
continue
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(self._flatten_dict(v, new_key, sep).items())
elif isinstance(v, (list, tuple)):
if all([isinstance(x, basestring) for x in v]):
items.append((new_key, v))
else:
items.append((new_key, v))
return dict(items)
return inv
def cache_available(cache_item, config):
''' checks if we have a 'fresh' cache available for item requested '''
if config.has_option('defaults', 'cache_dir'):
dpath = config.get('defaults', 'cache_dir')
def _get_obj_info(self, obj, depth=99, seen=None):
'''
Recursively build a data structure for the given pSphere object (depth
only applies to ManagedObject instances).
'''
seen = seen or set()
if isinstance(obj, ManagedObject):
try:
obj_unicode = unicode(getattr(obj, 'name'))
except AttributeError:
obj_unicode = ()
if obj in seen:
return obj_unicode
seen.add(obj)
if depth <= 0:
return obj_unicode
d = {}
for attr in dir(obj):
if attr.startswith('_'):
continue
try:
val = getattr(obj, attr)
obj_info = self._get_obj_info(val, depth - 1, seen)
if obj_info != ():
d[attr] = obj_info
except Exception, e:
pass
return d
elif isinstance(obj, SudsObject):
d = {}
for key, val in iter(obj):
obj_info = self._get_obj_info(val, depth, seen)
if obj_info != ():
d[key] = obj_info
return d
elif isinstance(obj, (list, tuple)):
l = []
for val in iter(obj):
obj_info = self._get_obj_info(val, depth, seen)
if obj_info != ():
l.append(obj_info)
return l
elif isinstance(obj, (type(None), bool, int, long, float, basestring)):
return obj
else:
return ()
def _get_host_info(self, host, prefix='vmware'):
'''
Return a flattened dict with info about the given host system.
'''
host_info = {
'name': host.name,
'tag': host.tag,
'datastores': self._get_obj_info(host.datastore, depth=0),
'networks': self._get_obj_info(host.network, depth=0),
'vms': self._get_obj_info(host.vm, depth=0),
}
for k, v in self._get_obj_info(host.summary, depth=0).items():
if isinstance(v, collections.MutableMapping):
for k2, v2 in v.items():
host_info[k2] = v2
elif k != 'host':
host_info[k] = v
try:
existing = os.stat( '/'.join([dpath,cache_item]))
except:
# cache doesn't exist or isn't accessible
return False
host_info['ipAddress'] = host.config.network.vnic[0].spec.ip.ipAddress
except Exception, e:
print >> sys.stderr, e
host_info = self._flatten_dict(host_info, prefix)
if ('%s_ipAddress' % prefix) in host_info:
host_info['ansible_ssh_host'] = host_info['%s_ipAddress' % prefix]
return host_info
if config.has_option('defaults', 'cache_max_age'):
maxage = config.get('defaults', 'cache_max_age')
def _get_vm_info(self, vm, prefix='vmware'):
'''
Return a flattened dict with info about the given virtual machine.
'''
vm_info = {
'name': vm.name,
'tag': vm.tag,
'datastores': self._get_obj_info(vm.datastore, depth=0),
'networks': self._get_obj_info(vm.network, depth=0),
'resourcePool': self._get_obj_info(vm.resourcePool, depth=0),
'guestState': vm.guest.guestState,
}
for k, v in self._get_obj_info(vm.summary, depth=0).items():
if isinstance(v, collections.MutableMapping):
for k2, v2 in v.items():
if k2 == 'host':
k2 = 'hostSystem'
vm_info[k2] = v2
elif k != 'vm':
vm_info[k] = v
vm_info = self._flatten_dict(vm_info, prefix)
if ('%s_ipAddress' % prefix) in vm_info:
vm_info['ansible_ssh_host'] = vm_info['%s_ipAddress' % prefix]
return vm_info
if (existing.st_mtime - int(time.time())) <= maxage:
return True
def _add_host(self, inv, parent_group, host_name):
'''
Add the host to the parent group in the given inventory.
'''
p_group = inv.setdefault(parent_group, [])
if isinstance(p_group, dict):
group_hosts = p_group.setdefault('hosts', [])
else:
group_hosts = p_group
if host_name not in group_hosts:
group_hosts.append(host_name)
return False
def _add_child(self, inv, parent_group, child_group):
'''
Add a child group to a parent group in the given inventory.
'''
if parent_group != 'all':
p_group = inv.setdefault(parent_group, {})
if not isinstance(p_group, dict):
inv[parent_group] = {'hosts': p_group}
p_group = inv[parent_group]
group_children = p_group.setdefault('children', [])
if child_group not in group_children:
group_children.append(child_group)
inv.setdefault(child_group, [])
def get_host_info(host):
''' Get variables about a specific host '''
def get_inventory(self, meta_hostvars=True):
'''
Reads the inventory from cache or VMware API via pSphere.
'''
# Use different cache names for guests only vs. all hosts.
if self.guests_only:
cache_name = '__inventory_guests__'
else:
cache_name = '__inventory_all__'
hostinfo = {
'vmware_name' : host.name,
'vmware_tag' : host.tag,
'vmware_parent': host.parent.name,
}
for k in host.capability.__dict__.keys():
if k.startswith('_'):
continue
try:
hostinfo['vmware_' + k] = str(host.capability[k])
except:
continue
inv = self._get_cache(cache_name, None)
if inv is not None:
return inv
return hostinfo
inv = {'all': {'hosts': []}}
if meta_hostvars:
inv['_meta'] = {'hostvars': {}}
def get_inventory(client, config):
''' Reads the inventory from cache or vmware api '''
if cache_available('inventory', config):
inv = get_cache('inventory',config)
else:
inv= { 'all': {'hosts': []}, '_meta': { 'hostvars': {} } }
default_group = os.path.basename(sys.argv[0]).rstrip('.py')
if config.has_option('defaults', 'guests_only'):
guests_only = config.get('defaults', 'guests_only')
else:
guests_only = True
if not guests_only:
if config.has_option('defaults','hw_group'):
hw_group = config.get('defaults','hw_group')
if not self.guests_only:
if self.config.has_option('defaults', 'hw_group'):
hw_group = self.config.get('defaults', 'hw_group')
else:
hw_group = default_group + '_hw'
inv[hw_group] = []
if config.has_option('defaults','vm_group'):
vm_group = config.get('defaults','vm_group')
if self.config.has_option('defaults', 'vm_group'):
vm_group = self.config.get('defaults', 'vm_group')
else:
vm_group = default_group + '_vm'
inv[vm_group] = []
# Loop through physical hosts:
hosts = HostSystem.all(client)
for host in hosts:
if not guests_only:
inv['all']['hosts'].append(host.name)
inv[hw_group].append(host.name)
if host.tag:
taggroup = 'vmware_' + host.tag
if taggroup in inv:
inv[taggroup].append(host.name)
else:
inv[taggroup] = [ host.name ]
for host in HostSystem.all(self.client):
inv['_meta']['hostvars'][host.name] = get_host_info(host)
save_cache(vm.name, inv['_meta']['hostvars'][host.name], config)
if not self.guests_only:
self._add_host(inv, 'all', host.name)
self._add_host(inv, hw_group, host.name)
if host.tag: # FIXME: Is this always a string?
host_tag = 'vmware_%s' % host.tag
self._add_host(inv, host_tag, host.name)
host_info = self._get_host_info(host)
if meta_hostvars:
inv['_meta']['hostvars'][host.name] = host_info
self._put_cache(host.name, host_info)
# Loop through all VMs on physical host.
for vm in host.vm:
inv['all']['hosts'].append(vm.name)
inv[vm_group].append(vm.name)
for tag in vm.tag:
taggroup = 'vmware_' + tag.key.lower()
if taggroup in inv:
inv[taggroup].append(vm.name)
else:
inv[taggroup] = [ vm.name ]
self._add_host(inv, 'all', vm.name)
self._add_host(inv, vm_group, vm.name)
if vm.tag: # FIXME: Is this always a string?
vm_tag = 'vmware_%s' % vm.tag
self._add_host(inv, vm_tag, vm.name)
vm_info = self._get_vm_info(vm)
if meta_hostvars:
inv['_meta']['hostvars'][vm.name] = vm_info
self._put_cache(vm.name, vm_info)
inv['_meta']['hostvars'][vm.name] = get_host_info(host)
save_cache(vm.name, inv['_meta']['hostvars'][vm.name], config)
# Group by resource pool.
vm_resourcePool = vm_info.get('vmware_resourcePool', None)
if vm_resourcePool:
self._add_child(inv, vm_group, 'resource_pools')
self._add_child(inv, 'resource_pools', vm_resourcePool)
self._add_host(inv, vm_resourcePool, vm.name)
save_cache('inventory', inv, config)
return json.dumps(inv)
# Group by datastore.
for vm_datastore in vm_info.get('vmware_datastores', []):
self._add_child(inv, vm_group, 'datastores')
self._add_child(inv, 'datastores', vm_datastore)
self._add_host(inv, vm_datastore, vm.name)
def get_single_host(client, config, hostname):
# Group by network.
for vm_network in vm_info.get('vmware_networks', []):
self._add_child(inv, vm_group, 'networks')
self._add_child(inv, 'networks', vm_network)
self._add_host(inv, vm_network, vm.name)
inv = {}
# Group by guest OS.
vm_guestId = vm_info.get('vmware_guestId', None)
if vm_guestId:
self._add_child(inv, vm_group, 'guests')
self._add_child(inv, 'guests', vm_guestId)
self._add_host(inv, vm_guestId, vm.name)
if cache_available(hostname, config):
inv = get_cache(hostname,config)
self._put_cache(cache_name, inv)
return inv
def get_host(self, hostname):
'''
Read info about a specific host or VM from cache or VMware API.
'''
inv = self._get_cache(hostname, None)
if inv is not None:
return inv
if not self.guests_only:
try:
host = HostSystem.get(self.client, name=hostname)
inv = self._get_host_info(host)
except ObjectNotFoundError:
pass
if inv is None:
try:
vm = VirtualMachine.get(self.client, name=hostname)
inv = self._get_vm_info(vm)
except ObjectNotFoundError:
pass
if inv is not None:
self._put_cache(hostname, inv)
return inv or {}
def main():
parser = optparse.OptionParser()
parser.add_option('--list', action='store_true', dest='list',
default=False, help='Output inventory groups and hosts')
parser.add_option('--host', dest='host', default=None, metavar='HOST',
help='Output variables only for the given hostname')
# Additional options for use when running the script standalone, but never
# used by Ansible.
parser.add_option('--pretty', action='store_true', dest='pretty',
default=False, help='Output nicely-formatted JSON')
parser.add_option('--include-host-systems', action='store_true',
dest='include_host_systems', default=False,
help='Include host systems in addition to VMs')
parser.add_option('--no-meta-hostvars', action='store_false',
dest='meta_hostvars', default=True,
help='Exclude [\'_meta\'][\'hostvars\'] with --list')
options, args = parser.parse_args()
if options.include_host_systems:
vmware_inventory = VMwareInventory(guests_only=False)
else:
hosts = HostSystem.all(client) #TODO: figure out single host getter
for host in hosts:
if hostname == host.name:
inv = get_host_info(host)
break
for vm in host.vm:
if hostname == vm.name:
inv = get_host_info(host)
break
save_cache(hostname,inv,config)
vmware_inventory = VMwareInventory()
if options.host is not None:
inventory = vmware_inventory.get_host(options.host)
else:
inventory = vmware_inventory.get_inventory(options.meta_hostvars)
json_kwargs = {}
if options.pretty:
json_kwargs.update({'indent': 4, 'sort_keys': True})
json.dump(inventory, sys.stdout, **json_kwargs)
return json.dumps(inv)
if __name__ == '__main__':
inventory = {}
hostname = None
if len(sys.argv) > 1:
if sys.argv[1] == "--host":
hostname = sys.argv[2]
# Read config
config = ConfigParser.SafeConfigParser(
defaults={'host': '', 'user': '', 'password': ''},
)
for section in ('auth', 'defaults'):
config.add_section(section)
for configfilename in [os.path.abspath(sys.argv[0]).rstrip('.py') + '.ini', 'vmware.ini']:
if os.path.exists(configfilename):
config.read(configfilename)
break
auth_host, auth_user, auth_password = None, None, None
# Read our authentication information from the INI file, if it exists.
try:
auth_host = config.get('auth', 'host')
auth_user = config.get('auth', 'user')
auth_password = config.get('auth', 'password')
except Exception:
pass
# If any of the VMware environment variables are set, they trump
# the INI configuration.
if 'VMWARE_HOST' in os.environ:
auth_host = os.environ['VMWARE_HOST']
if 'VMWARE_USER' in os.environ:
auth_user = os.environ['VMWARE_USER']
if 'VMWARE_PASSWORD' in os.environ:
auth_password = os.environ['VMWARE_PASSWORD']
# Create the VMware client.
client = Client(auth_host, auth_user, auth_password)
# Actually do the work.
if hostname is None:
inventory = get_inventory(client, config)
else:
inventory = get_single_host(client, config, hostname)
# Return to Ansible.
print inventory
main()

6
awx/settings/defaults.py
View File

@ -399,11 +399,11 @@ VMWARE_REGIONS_BLACKLIST = []
# Inventory variable name/values for determining whether a host is
# active in vSphere.
VMWARE_ENABLED_VAR = 'status'
VMWARE_ENABLED_VALUE = 'POWERED ON'
VMWARE_ENABLED_VAR = 'vmware_powerState'
VMWARE_ENABLED_VALUE = 'poweredOn'
# Inventory variable name containing the unique instance ID.
VMWARE_INSTANCE_ID_VAR = 'guest_id'
VMWARE_INSTANCE_ID_VAR = 'vmware_uuid'
# Filter for allowed group and host names when importing inventory
# from EC2.

5
awx/settings/local_settings.py.example
View File

@ -484,3 +484,8 @@ TEST_AWS_REGIONS = 'all'
TEST_RACKSPACE_USERNAME = ''
TEST_RACKSPACE_API_KEY = ''
TEST_RACKSPACE_REGIONS = 'all'
# VMware credentials
TEST_VMWARE_HOST = ''
TEST_VMWARE_USER = ''
TEST_VMWARE_PASSWORD = ''