mirror of
https://github.com/ansible/awx.git
synced 2024-11-01 16:51:11 +03:00
Import latest ec2.py and fix an issue
Upgrading ec2.py to fix an issue where the module wasn't obeying "replace_dash_in_groups" property.
This commit is contained in:
Matthew Jones
parent
8a884894c5
commit
bdf5ee1fea
@ -857,7 +857,7 @@ class Command(NoArgsCommand):
|
||||
del_pks = del_host_pks[offset:(offset + self._batch_size)]
|
||||
for db_host in db_hosts.filter(pk__in=del_pks):
|
||||
group_host_count += 1
|
||||
if db_host not in db_group.hosts:
|
||||
if db_host not in db_group.hosts.all():
|
||||
continue
|
||||
db_group.hosts.remove(db_host)
|
||||
self.logger.info('Host "%s" removed from group "%s"',
|
||||
|
||||
@ -141,6 +141,7 @@ except ImportError:
|
||||
|
||||
|
||||
class Ec2Inventory(object):
|
||||
|
||||
def _empty_inventory(self):
|
||||
return {"_meta" : {"hostvars" : {}}}
|
||||
|
||||
@ -157,6 +158,9 @@ class Ec2Inventory(object):
|
||||
# Boto profile to use (if any)
|
||||
self.boto_profile = None
|
||||
|
||||
# AWS credentials.
|
||||
self.credentials = {}
|
||||
|
||||
# Read settings and parse CLI arguments
|
||||
self.parse_cli_args()
|
||||
self.read_settings()
|
||||
@ -224,7 +228,7 @@ class Ec2Inventory(object):
|
||||
configRegions_exclude = config.get('ec2', 'regions_exclude')
|
||||
if (configRegions == 'all'):
|
||||
if self.eucalyptus_host:
|
||||
self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name)
|
||||
self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name, **self.credentials)
|
||||
else:
|
||||
for regionInfo in ec2.regions():
|
||||
if regionInfo.name not in configRegions_exclude:
|
||||
@ -236,6 +240,19 @@ class Ec2Inventory(object):
|
||||
self.destination_variable = config.get('ec2', 'destination_variable')
|
||||
self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable')
|
||||
|
||||
if config.has_option('ec2', 'hostname_variable'):
|
||||
self.hostname_variable = config.get('ec2', 'hostname_variable')
|
||||
else:
|
||||
self.hostname_variable = None
|
||||
|
||||
if config.has_option('ec2', 'destination_format') and \
|
||||
config.has_option('ec2', 'destination_format_tags'):
|
||||
self.destination_format = config.get('ec2', 'destination_format')
|
||||
self.destination_format_tags = config.get('ec2', 'destination_format_tags').split(',')
|
||||
else:
|
||||
self.destination_format = None
|
||||
self.destination_format_tags = None
|
||||
|
||||
# Route53
|
||||
self.route53_enabled = config.getboolean('ec2', 'route53')
|
||||
self.route53_excluded_zones = []
|
||||
@ -310,6 +327,29 @@ class Ec2Inventory(object):
|
||||
if config.has_option('ec2', 'boto_profile') and not self.boto_profile:
|
||||
self.boto_profile = config.get('ec2', 'boto_profile')
|
||||
|
||||
# AWS credentials (prefer environment variables)
|
||||
if not (self.boto_profile or os.environ.get('AWS_ACCESS_KEY_ID') or
|
||||
os.environ.get('AWS_PROFILE')):
|
||||
if config.has_option('credentials', 'aws_access_key_id'):
|
||||
aws_access_key_id = config.get('credentials', 'aws_access_key_id')
|
||||
else:
|
||||
aws_access_key_id = None
|
||||
if config.has_option('credentials', 'aws_secret_access_key'):
|
||||
aws_secret_access_key = config.get('credentials', 'aws_secret_access_key')
|
||||
else:
|
||||
aws_secret_access_key = None
|
||||
if config.has_option('credentials', 'aws_security_token'):
|
||||
aws_security_token = config.get('credentials', 'aws_security_token')
|
||||
else:
|
||||
aws_security_token = None
|
||||
if aws_access_key_id:
|
||||
self.credentials = {
|
||||
'aws_access_key_id': aws_access_key_id,
|
||||
'aws_secret_access_key': aws_secret_access_key
|
||||
}
|
||||
if aws_security_token:
|
||||
self.credentials['security_token'] = aws_security_token
|
||||
|
||||
# Cache related
|
||||
cache_dir = os.path.expanduser(config.get('ec2', 'cache_path'))
|
||||
if self.boto_profile:
|
||||
@ -317,10 +357,22 @@ class Ec2Inventory(object):
|
||||
if not os.path.exists(cache_dir):
|
||||
os.makedirs(cache_dir)
|
||||
|
||||
self.cache_path_cache = cache_dir + "/ansible-ec2.cache"
|
||||
self.cache_path_index = cache_dir + "/ansible-ec2.index"
|
||||
cache_name = 'ansible-ec2'
|
||||
aws_profile = lambda: (self.boto_profile or
|
||||
os.environ.get('AWS_PROFILE') or
|
||||
os.environ.get('AWS_ACCESS_KEY_ID') or
|
||||
self.credentials.get('aws_access_key_id', None))
|
||||
if aws_profile():
|
||||
cache_name = '%s-%s' % (cache_name, aws_profile())
|
||||
self.cache_path_cache = cache_dir + "/%s.cache" % cache_name
|
||||
self.cache_path_index = cache_dir + "/%s.index" % cache_name
|
||||
self.cache_max_age = config.getint('ec2', 'cache_max_age')
|
||||
|
||||
if config.has_option('ec2', 'expand_csv_tags'):
|
||||
self.expand_csv_tags = config.getboolean('ec2', 'expand_csv_tags')
|
||||
else:
|
||||
self.expand_csv_tags = False
|
||||
|
||||
# Configure nested groups instead of flat namespace.
|
||||
if config.has_option('ec2', 'nested_groups'):
|
||||
self.nested_groups = config.getboolean('ec2', 'nested_groups')
|
||||
@ -382,7 +434,10 @@ class Ec2Inventory(object):
|
||||
# Instance filters (see boto and EC2 API docs). Ignore invalid filters.
|
||||
self.ec2_instance_filters = defaultdict(list)
|
||||
if config.has_option('ec2', 'instance_filters'):
|
||||
for instance_filter in config.get('ec2', 'instance_filters', '').split(','):
|
||||
|
||||
filters = [f for f in config.get('ec2', 'instance_filters').split(',') if f]
|
||||
|
||||
for instance_filter in filters:
|
||||
instance_filter = instance_filter.strip()
|
||||
if not instance_filter or '=' not in instance_filter:
|
||||
continue
|
||||
@ -401,7 +456,7 @@ class Ec2Inventory(object):
|
||||
help='Get all the variables about a specific instance')
|
||||
parser.add_argument('--refresh-cache', action='store_true', default=False,
|
||||
help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)')
|
||||
parser.add_argument('--boto-profile', action='store',
|
||||
parser.add_argument('--profile', '--boto-profile', action='store', dest='boto_profile',
|
||||
help='Use boto profile for connections to EC2')
|
||||
self.args = parser.parse_args()
|
||||
|
||||
@ -426,7 +481,7 @@ class Ec2Inventory(object):
|
||||
def connect(self, region):
|
||||
''' create connection to api server'''
|
||||
if self.eucalyptus:
|
||||
conn = boto.connect_euca(host=self.eucalyptus_host)
|
||||
conn = boto.connect_euca(host=self.eucalyptus_host, **self.credentials)
|
||||
conn.APIVersion = '2010-08-31'
|
||||
else:
|
||||
conn = self.connect_to_aws(ec2, region)
|
||||
@ -440,7 +495,7 @@ class Ec2Inventory(object):
|
||||
return connect_args
|
||||
|
||||
def connect_to_aws(self, module, region):
|
||||
connect_args = {}
|
||||
connect_args = self.credentials
|
||||
|
||||
# only pass the profile name if it's set (as it is not supported by older boto versions)
|
||||
if self.boto_profile:
|
||||
@ -466,15 +521,27 @@ class Ec2Inventory(object):
|
||||
else:
|
||||
reservations = conn.get_all_instances()
|
||||
|
||||
# Pull the tags back in a second step
|
||||
# AWS are on record as saying that the tags fetched in the first `get_all_instances` request are not
|
||||
# reliable and may be missing, and the only way to guarantee they are there is by calling `get_all_tags`
|
||||
instance_ids = []
|
||||
for reservation in reservations:
|
||||
instance_ids.extend([instance.id for instance in reservation.instances])
|
||||
tags = conn.get_all_tags(filters={'resource-type': 'instance', 'resource-id': instance_ids})
|
||||
tags_by_instance_id = defaultdict(dict)
|
||||
for tag in tags:
|
||||
tags_by_instance_id[tag.res_id][tag.name] = tag.value
|
||||
|
||||
for reservation in reservations:
|
||||
for instance in reservation.instances:
|
||||
instance.tags = tags_by_instance_id[instance.id]
|
||||
self.add_instance(instance, region)
|
||||
|
||||
except boto.exception.BotoServerError as e:
|
||||
if e.error_code == 'AuthFailure':
|
||||
error = self.get_auth_error_message()
|
||||
else:
|
||||
backend = 'Eucalyptus' if self.eucalyptus else 'AWS'
|
||||
backend = 'Eucalyptus' if self.eucalyptus else 'AWS'
|
||||
error = "Error connecting to %s backend.\n%s" % (backend, e.message)
|
||||
self.fail_with_error(error, 'getting EC2 instances')
|
||||
|
||||
@ -485,9 +552,14 @@ class Ec2Inventory(object):
|
||||
try:
|
||||
conn = self.connect_to_aws(rds, region)
|
||||
if conn:
|
||||
instances = conn.get_all_dbinstances()
|
||||
for instance in instances:
|
||||
self.add_rds_instance(instance, region)
|
||||
marker = None
|
||||
while True:
|
||||
instances = conn.get_all_dbinstances(marker=marker)
|
||||
marker = instances.marker
|
||||
for instance in instances:
|
||||
self.add_rds_instance(instance, region)
|
||||
if not marker:
|
||||
break
|
||||
except boto.exception.BotoServerError as e:
|
||||
error = e.reason
|
||||
|
||||
@ -609,7 +681,9 @@ class Ec2Inventory(object):
|
||||
return
|
||||
|
||||
# Select the best destination address
|
||||
if instance.subnet_id:
|
||||
if self.destination_format and self.destination_format_tags:
|
||||
dest = self.destination_format.format(*[ getattr(instance, 'tags').get(tag, '') for tag in self.destination_format_tags ])
|
||||
elif instance.subnet_id:
|
||||
dest = getattr(instance, self.vpc_destination_variable, None)
|
||||
if dest is None:
|
||||
dest = getattr(instance, 'tags').get(self.vpc_destination_variable, None)
|
||||
@ -622,32 +696,46 @@ class Ec2Inventory(object):
|
||||
# Skip instances we cannot address (e.g. private VPC subnet)
|
||||
return
|
||||
|
||||
# Set the inventory name
|
||||
hostname = None
|
||||
if self.hostname_variable:
|
||||
if self.hostname_variable.startswith('tag_'):
|
||||
hostname = instance.tags.get(self.hostname_variable[4:], None)
|
||||
else:
|
||||
hostname = getattr(instance, self.hostname_variable)
|
||||
|
||||
# If we can't get a nice hostname, use the destination address
|
||||
if not hostname:
|
||||
hostname = dest
|
||||
else:
|
||||
hostname = self.to_safe(hostname).lower()
|
||||
|
||||
# if we only want to include hosts that match a pattern, skip those that don't
|
||||
if self.pattern_include and not self.pattern_include.match(dest):
|
||||
if self.pattern_include and not self.pattern_include.match(hostname):
|
||||
return
|
||||
|
||||
# if we need to exclude hosts that match a pattern, skip those
|
||||
if self.pattern_exclude and self.pattern_exclude.match(dest):
|
||||
if self.pattern_exclude and self.pattern_exclude.match(hostname):
|
||||
return
|
||||
|
||||
# Add to index
|
||||
self.index[dest] = [region, instance.id]
|
||||
self.index[hostname] = [region, instance.id]
|
||||
|
||||
# Inventory: Group by instance ID (always a group of 1)
|
||||
if self.group_by_instance_id:
|
||||
self.inventory[instance.id] = [dest]
|
||||
self.inventory[instance.id] = [hostname]
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'instances', instance.id)
|
||||
|
||||
# Inventory: Group by region
|
||||
if self.group_by_region:
|
||||
self.push(self.inventory, region, dest)
|
||||
self.push(self.inventory, region, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'regions', region)
|
||||
|
||||
# Inventory: Group by availability zone
|
||||
if self.group_by_availability_zone:
|
||||
self.push(self.inventory, instance.placement, dest)
|
||||
self.push(self.inventory, instance.placement, hostname)
|
||||
if self.nested_groups:
|
||||
if self.group_by_region:
|
||||
self.push_group(self.inventory, region, instance.placement)
|
||||
@ -656,28 +744,28 @@ class Ec2Inventory(object):
|
||||
# Inventory: Group by Amazon Machine Image (AMI) ID
|
||||
if self.group_by_ami_id:
|
||||
ami_id = self.to_safe(instance.image_id)
|
||||
self.push(self.inventory, ami_id, dest)
|
||||
self.push(self.inventory, ami_id, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'images', ami_id)
|
||||
|
||||
# Inventory: Group by instance type
|
||||
if self.group_by_instance_type:
|
||||
type_name = self.to_safe('type_' + instance.instance_type)
|
||||
self.push(self.inventory, type_name, dest)
|
||||
self.push(self.inventory, type_name, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'types', type_name)
|
||||
|
||||
# Inventory: Group by key pair
|
||||
if self.group_by_key_pair and instance.key_name:
|
||||
key_name = self.to_safe('key_' + instance.key_name)
|
||||
self.push(self.inventory, key_name, dest)
|
||||
self.push(self.inventory, key_name, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'keys', key_name)
|
||||
|
||||
# Inventory: Group by VPC
|
||||
if self.group_by_vpc_id and instance.vpc_id:
|
||||
vpc_id_name = self.to_safe('vpc_id_' + instance.vpc_id)
|
||||
self.push(self.inventory, vpc_id_name, dest)
|
||||
self.push(self.inventory, vpc_id_name, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'vpcs', vpc_id_name)
|
||||
|
||||
@ -686,44 +774,51 @@ class Ec2Inventory(object):
|
||||
try:
|
||||
for group in instance.groups:
|
||||
key = self.to_safe("security_group_" + group.name)
|
||||
self.push(self.inventory, key, dest)
|
||||
self.push(self.inventory, key, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'security_groups', key)
|
||||
except AttributeError:
|
||||
self.fail_with_error('\n'.join(['Package boto seems a bit older.',
|
||||
self.fail_with_error('\n'.join(['Package boto seems a bit older.',
|
||||
'Please upgrade boto >= 2.3.0.']))
|
||||
|
||||
# Inventory: Group by tag keys
|
||||
if self.group_by_tag_keys:
|
||||
for k, v in instance.tags.items():
|
||||
if v:
|
||||
key = self.to_safe("tag_" + k + "=" + v)
|
||||
if self.expand_csv_tags and v and ',' in v:
|
||||
values = map(lambda x: x.strip(), v.split(','))
|
||||
else:
|
||||
key = self.to_safe("tag_" + k)
|
||||
self.push(self.inventory, key, dest)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k))
|
||||
values = [v]
|
||||
|
||||
for v in values:
|
||||
if v:
|
||||
self.push_group(self.inventory, self.to_safe("tag_" + k), key)
|
||||
key = self.to_safe("tag_" + k + "=" + v)
|
||||
else:
|
||||
key = self.to_safe("tag_" + k)
|
||||
self.push(self.inventory, key, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k))
|
||||
if v:
|
||||
self.push_group(self.inventory, self.to_safe("tag_" + k), key)
|
||||
|
||||
# Inventory: Group by Route53 domain names if enabled
|
||||
if self.route53_enabled and self.group_by_route53_names:
|
||||
route53_names = self.get_instance_route53_names(instance)
|
||||
for name in route53_names:
|
||||
self.push(self.inventory, name, dest)
|
||||
self.push(self.inventory, name, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'route53', name)
|
||||
|
||||
# Global Tag: instances without tags
|
||||
if self.group_by_tag_none and len(instance.tags) == 0:
|
||||
self.push(self.inventory, 'tag_none', dest)
|
||||
self.push(self.inventory, 'tag_none', hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'tags', 'tag_none')
|
||||
|
||||
# Global Tag: tag all EC2 instances
|
||||
self.push(self.inventory, 'ec2', dest)
|
||||
self.push(self.inventory, 'ec2', hostname)
|
||||
|
||||
self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance)
|
||||
self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance)
|
||||
self.inventory["_meta"]["hostvars"][hostname]['ansible_ssh_host'] = dest
|
||||
|
||||
|
||||
def add_rds_instance(self, instance, region):
|
||||
@ -741,24 +836,38 @@ class Ec2Inventory(object):
|
||||
# Skip instances we cannot address (e.g. private VPC subnet)
|
||||
return
|
||||
|
||||
# Set the inventory name
|
||||
hostname = None
|
||||
if self.hostname_variable:
|
||||
if self.hostname_variable.startswith('tag_'):
|
||||
hostname = instance.tags.get(self.hostname_variable[4:], None)
|
||||
else:
|
||||
hostname = getattr(instance, self.hostname_variable)
|
||||
|
||||
# If we can't get a nice hostname, use the destination address
|
||||
if not hostname:
|
||||
hostname = dest
|
||||
|
||||
hostname = self.to_safe(hostname).lower()
|
||||
|
||||
# Add to index
|
||||
self.index[dest] = [region, instance.id]
|
||||
self.index[hostname] = [region, instance.id]
|
||||
|
||||
# Inventory: Group by instance ID (always a group of 1)
|
||||
if self.group_by_instance_id:
|
||||
self.inventory[instance.id] = [dest]
|
||||
self.inventory[instance.id] = [hostname]
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'instances', instance.id)
|
||||
|
||||
# Inventory: Group by region
|
||||
if self.group_by_region:
|
||||
self.push(self.inventory, region, dest)
|
||||
self.push(self.inventory, region, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'regions', region)
|
||||
|
||||
# Inventory: Group by availability zone
|
||||
if self.group_by_availability_zone:
|
||||
self.push(self.inventory, instance.availability_zone, dest)
|
||||
self.push(self.inventory, instance.availability_zone, hostname)
|
||||
if self.nested_groups:
|
||||
if self.group_by_region:
|
||||
self.push_group(self.inventory, region, instance.availability_zone)
|
||||
@ -767,14 +876,14 @@ class Ec2Inventory(object):
|
||||
# Inventory: Group by instance type
|
||||
if self.group_by_instance_type:
|
||||
type_name = self.to_safe('type_' + instance.instance_class)
|
||||
self.push(self.inventory, type_name, dest)
|
||||
self.push(self.inventory, type_name, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'types', type_name)
|
||||
|
||||
# Inventory: Group by VPC
|
||||
if self.group_by_vpc_id and instance.subnet_group and instance.subnet_group.vpc_id:
|
||||
vpc_id_name = self.to_safe('vpc_id_' + instance.subnet_group.vpc_id)
|
||||
self.push(self.inventory, vpc_id_name, dest)
|
||||
self.push(self.inventory, vpc_id_name, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'vpcs', vpc_id_name)
|
||||
|
||||
@ -783,31 +892,32 @@ class Ec2Inventory(object):
|
||||
try:
|
||||
if instance.security_group:
|
||||
key = self.to_safe("security_group_" + instance.security_group.name)
|
||||
self.push(self.inventory, key, dest)
|
||||
self.push(self.inventory, key, hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'security_groups', key)
|
||||
|
||||
except AttributeError:
|
||||
self.fail_with_error('\n'.join(['Package boto seems a bit older.',
|
||||
self.fail_with_error('\n'.join(['Package boto seems a bit older.',
|
||||
'Please upgrade boto >= 2.3.0.']))
|
||||
|
||||
|
||||
# Inventory: Group by engine
|
||||
if self.group_by_rds_engine:
|
||||
self.push(self.inventory, self.to_safe("rds_" + instance.engine), dest)
|
||||
self.push(self.inventory, self.to_safe("rds_" + instance.engine), hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'rds_engines', self.to_safe("rds_" + instance.engine))
|
||||
|
||||
# Inventory: Group by parameter group
|
||||
if self.group_by_rds_parameter_group:
|
||||
self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), dest)
|
||||
self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), hostname)
|
||||
if self.nested_groups:
|
||||
self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe("rds_parameter_group_" + instance.parameter_group.name))
|
||||
|
||||
# Global Tag: all RDS instances
|
||||
self.push(self.inventory, 'rds', dest)
|
||||
self.push(self.inventory, 'rds', hostname)
|
||||
|
||||
self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance)
|
||||
self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance)
|
||||
self.inventory["_meta"]["hostvars"][hostname]['ansible_ssh_host'] = dest
|
||||
|
||||
def add_elasticache_cluster(self, cluster, region):
|
||||
''' Adds an ElastiCache cluster to the inventory and index, as long as
|
||||
@ -1120,6 +1230,8 @@ class Ec2Inventory(object):
|
||||
instance_vars['ec2_placement'] = value.zone
|
||||
elif key == 'ec2_tags':
|
||||
for k, v in value.items():
|
||||
if self.expand_csv_tags and ',' in v:
|
||||
v = map(lambda x: x.strip(), v.split(','))
|
||||
key = self.to_safe('ec2_tag_' + k)
|
||||
instance_vars[key] = v
|
||||
elif key == 'ec2_groups':
|
||||
@ -1294,7 +1406,7 @@ class Ec2Inventory(object):
|
||||
def to_safe(self, word):
|
||||
''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups '''
|
||||
regex = "[^A-Za-z0-9\_"
|
||||
if self.replace_dash_in_groups:
|
||||
if not self.replace_dash_in_groups:
|
||||
regex += "\-"
|
||||
return re.sub(regex + "]", "_", word)
|
||||
|
||||
@ -1310,4 +1422,3 @@ class Ec2Inventory(object):
|
||||
|
||||
# Run the script
|
||||
Ec2Inventory()
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user