diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 816f3651ef..d0e1a68c9f 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -733,6 +733,7 @@ class Command(NoArgsCommand): db_group.children.remove(db_child) self.logger.info('Group "%s" removed from group "%s"', db_child.name, db_group.name) + # FIXME: Inventory source group relationships # Delete group/host relationships not present in imported data. db_hosts = db_group.hosts.filter(active=True) del_host_pks = set(db_hosts.values_list('pk', flat=True)) @@ -804,6 +805,12 @@ class Command(NoArgsCommand): queries_before = len(connection.queries) inv_src_group = self.inventory_source.group all_group_names = sorted(self.all_group.all_groups.keys()) + root_group_names = set() + for k,v in self.all_group.all_groups.items(): + if not v.parents: + root_group_names.add(k) + if len(v.parents) == 1 and v.parents[0].name == 'all': + root_group_names.add(k) existing_group_names = set() for offset in xrange(0, len(all_group_names), self._batch_size): group_names = all_group_names[offset:(offset + self._batch_size)] @@ -824,7 +831,7 @@ class Command(NoArgsCommand): else: self.logger.info('Group "%s" variables unmodified', group.name) existing_group_names.add(group.name) - if inv_src_group and inv_src_group != group: + if inv_src_group and inv_src_group != group and group.name in root_group_names: self._batch_add_m2m(inv_src_group.children, group) self._batch_add_m2m(self.inventory_source.groups, group) for group_name in all_group_names: @@ -836,7 +843,7 @@ class Command(NoArgsCommand): #group.inventory_source InventorySource.objects.create(group=group, inventory=self.inventory, name=('%s (%s)' % (group_name, self.inventory.name))) self.logger.info('Group "%s" added', group.name) - if inv_src_group: + if inv_src_group and group_name in root_group_names: self._batch_add_m2m(inv_src_group.children, group) self._batch_add_m2m(self.inventory_source.groups, group) if inv_src_group: @@ -908,7 +915,6 @@ class Command(NoArgsCommand): if self.inventory_source.group: self._batch_add_m2m(self.inventory_source.group.hosts, db_host) self._batch_add_m2m(self.inventory_source.hosts, db_host) - #host.update_computed_fields(False, False) def _create_update_hosts(self): ''' @@ -995,7 +1001,6 @@ class Command(NoArgsCommand): if self.inventory_source.group: self._batch_add_m2m(self.inventory_source.group.hosts, db_host) self._batch_add_m2m(self.inventory_source.hosts, db_host) - #host.update_computed_fields(False, False) if self.inventory_source.group: self._batch_add_m2m(self.inventory_source.group.hosts, flush=True) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 3a0faf0fe8..b36eac392e 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -806,6 +806,7 @@ class RunInventoryUpdate(BaseTask): ec2_opts.setdefault('destination_variable', 'public_dns_name') ec2_opts.setdefault('vpc_destination_variable', 'ip_address') ec2_opts.setdefault('route53', 'False') + ec2_opts.setdefault('nested_groups', 'True') ec2_opts['cache_path'] = tempfile.mkdtemp(prefix='awx_ec2_') ec2_opts['cache_max_age'] = '300' for k,v in ec2_opts.items(): diff --git a/awx/main/tests/inventory.py b/awx/main/tests/inventory.py index 10cdc1e942..7033f14b6a 100644 --- a/awx/main/tests/inventory.py +++ b/awx/main/tests/inventory.py @@ -1398,7 +1398,7 @@ class InventoryUpdatesTest(BaseTransactionTest): self.group = group inventory_source = self.update_inventory_source(self.group, source='ec2', credential=credential, source_regions=source_regions, - source_vars='---') + source_vars='---\n\nnested_groups: false\n') # Check first without instance_id set (to import by name only). with self.settings(EC2_INSTANCE_ID_VAR=''): self.check_inventory_source(inventory_source) @@ -1427,6 +1427,58 @@ class InventoryUpdatesTest(BaseTransactionTest): # Verify that main group is in top level groups (hasn't been added as # its own child). self.assertTrue(self.group in self.inventory.root_groups) + + def test_update_from_ec2_with_nested_groups(self): + source_username = getattr(settings, 'TEST_AWS_ACCESS_KEY_ID', '') + source_password = getattr(settings, 'TEST_AWS_SECRET_ACCESS_KEY', '') + source_regions = getattr(settings, 'TEST_AWS_REGIONS', 'all') + if not all([source_username, source_password]): + self.skipTest('no test ec2 credentials defined!') + credential = Credential.objects.create(kind='aws', + user=self.super_django_user, + username=source_username, + password=source_password) + group = self.group + group.name = 'AWS Inventory' + group.save() + self.group = group + inventory_source = self.update_inventory_source(self.group, + source='ec2', credential=credential, source_regions=source_regions, + source_vars='---') # nested_groups is true by default. + self.check_inventory_source(inventory_source) + # Manually disable all hosts, verify a new update re-enables them. + for host in self.inventory.hosts.all(): + host.enabled = False + host.save() + self.check_inventory_source(inventory_source, initial=False) + # Verify that main group is in top level groups (hasn't been added as + # its own child). + self.assertTrue(self.group in self.inventory.root_groups) + # Verify that returned groups are nested: + child_names = self.group.children.values_list('name', flat=True) + for name in child_names: + self.assertFalse(name.startswith('us-')) + self.assertFalse(name.startswith('type_')) + self.assertFalse(name.startswith('key_')) + self.assertFalse(name.startswith('security_group_')) + self.assertFalse(name.startswith('tag_')) + self.assertTrue('ec2' in child_names) + self.assertTrue('regions' in child_names) + self.assertTrue('types' in child_names) + self.assertTrue('keys' in child_names) + self.assertTrue('security_groups' in child_names) + self.assertTrue('tags' in child_names) + # Print out group/host tree for debugging. + return + print + def draw_tree(g, d=0): + print (' ' * d) + '+ ' + g.name + for h in g.hosts.order_by('name'): + print (' ' * d) + ' - ' + h.name + for c in g.children.order_by('name'): + draw_tree(c, d+1) + for g in self.inventory.root_groups.order_by('name'): + draw_tree(g) def test_update_from_rax(self): source_username = getattr(settings, 'TEST_RACKSPACE_USERNAME', '') diff --git a/awx/plugins/inventory/ec2.ini.example b/awx/plugins/inventory/ec2.ini.example index b931c4a7da..98856dad51 100644 --- a/awx/plugins/inventory/ec2.ini.example +++ b/awx/plugins/inventory/ec2.ini.example @@ -52,3 +52,7 @@ cache_path = ~/.ansible/tmp # The number of seconds a cache file is considered valid. After this many # seconds, a new API call will be made, and the cache file will be updated. cache_max_age = 300 + +# For Ansible Tower, organize groups into a nested/hierarchy instead of a flat +# namespace. +nested_groups = True diff --git a/awx/plugins/inventory/ec2.py b/awx/plugins/inventory/ec2.py index 84841d3f09..283af2fb1f 100755 --- a/awx/plugins/inventory/ec2.py +++ b/awx/plugins/inventory/ec2.py @@ -230,8 +230,12 @@ class Ec2Inventory(object): self.cache_path_cache = cache_dir + "/ansible-ec2.cache" self.cache_path_index = cache_dir + "/ansible-ec2.index" self.cache_max_age = config.getint('ec2', 'cache_max_age') - + # Ansible Tower - configure nested groups instead of flat namespace. + if config.has_option('ec2', 'nested_groups'): + self.nested_groups = config.getboolean('ec2', 'nested_groups') + else: + self.nested_groups = False def parse_cli_args(self): ''' Command line argument processing ''' @@ -321,14 +325,14 @@ class Ec2Inventory(object): for instance in reservation.instances: return instance - def add_instance(self, instance, region): ''' Adds an instance to the inventory and index, as long as it is addressable ''' + # For Ansible Tower, return all instances regardless of state. # Only want running instances - if instance.state != 'running': - return + #if instance.state != 'running': + # return # Select the best destination address if instance.subnet_id: @@ -347,23 +351,36 @@ class Ec2Inventory(object): self.inventory[instance.id] = [dest] # Inventory: Group by region - self.push(self.inventory, region, dest) + if self.nested_groups: + self.push_group(self.inventory, 'regions', region) + else: + self.push(self.inventory, region, dest) # Inventory: Group by availability zone self.push(self.inventory, instance.placement, dest) + if self.nested_groups: + self.push_group(self.inventory, region, instance.placement) # Inventory: Group by instance type - self.push(self.inventory, self.to_safe('type_' + instance.instance_type), dest) + type_name = self.to_safe('type_' + instance.instance_type) + self.push(self.inventory, type_name, dest) + if self.nested_groups: + self.push_group(self.inventory, 'types', type_name) # Inventory: Group by key pair if instance.key_name: - self.push(self.inventory, self.to_safe('key_' + instance.key_name), dest) + key_name = self.to_safe('key_' + instance.key_name) + self.push(self.inventory, key_name, dest) + if self.nested_groups: + self.push_group(self.inventory, 'keys', key_name) # Inventory: Group by security group try: for group in instance.groups: key = self.to_safe("security_group_" + group.name) self.push(self.inventory, key, dest) + if self.nested_groups: + self.push_group(self.inventory, 'security_groups', key) except AttributeError: print 'Package boto seems a bit older.' print 'Please upgrade boto >= 2.3.0.' @@ -373,12 +390,17 @@ class Ec2Inventory(object): for k, v in instance.tags.iteritems(): key = self.to_safe("tag_" + k + "=" + v) self.push(self.inventory, key, dest) + if self.nested_groups: + self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k)) + self.push_group(self.inventory, self.to_safe("tag_" + k), key) # Inventory: Group by Route53 domain names if enabled if self.route53_enabled: route53_names = self.get_instance_route53_names(instance) for name in route53_names: self.push(self.inventory, name, dest) + if self.nested_groups: + self.push_group(self.inventory, 'route53', name) # Global Tag: tag all EC2 instances self.push(self.inventory, 'ec2', dest) @@ -561,6 +583,12 @@ class Ec2Inventory(object): else: my_dict[key] = [element] + def push_group(self, my_dict, key, element): + '''Push a group as a child of another group.''' + parent_group = my_dict.setdefault(key, {}) + child_groups = parent_group.setdefault('children', []) + if element not in child_groups: + child_groups.append(element) def get_inventory_from_cache(self): ''' Reads the inventory from the cache file and returns it as a JSON