From e01d98f2a2469e7d11713339fe5a4dbd4b707bf7 Mon Sep 17 00:00:00 2001 From: adamscmRH Date: Tue, 11 Jul 2017 12:09:25 -0400 Subject: [PATCH 1/5] Fix IntegrityError on Source Update --- awx/main/management/commands/inventory_import.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 691fbaca78..6b9026ac64 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -679,6 +679,7 @@ class Command(NoArgsCommand): logger.info('Host "%s" is now disabled', mem_host.name) self._batch_add_m2m(self.inventory_source.hosts, db_host) + @transaction.atomic def _create_update_hosts(self): ''' For each host in the local list, create it if it doesn't exist in the @@ -748,7 +749,7 @@ class Command(NoArgsCommand): if self.instance_id_var: instance_id = self._get_instance_id(mem_host.variables) host_attrs['instance_id'] = instance_id - db_host = self.inventory.hosts.create(**host_attrs) + db_host, createdHost = self.inventory.hosts.get_or_create(**host_attrs) if enabled is False: logger.info('Host "%s" added (disabled)', mem_host_name) else: From 58bd747cb199b629f1ec0e4c48883a8f22997610 Mon Sep 17 00:00:00 2001 From: adamscmRH Date: Tue, 11 Jul 2017 17:06:21 -0400 Subject: [PATCH 2/5] Fixes group & host integrity error --- awx/main/management/commands/inventory_import.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 6b9026ac64..984ccb11d2 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -421,6 +421,7 @@ class Command(NoArgsCommand): mem_host.instance_id = instance_id self.mem_instance_id_map[instance_id] = mem_host.name + @transaction.atomic def _delete_hosts(self): ''' For each host in the database that is NOT in the local list, delete @@ -462,6 +463,7 @@ class Command(NoArgsCommand): len(connection.queries) - queries_before, len(all_del_pks)) + @transaction.atomic def _delete_groups(self): ''' # If overwrite is set, for each group in the database that is NOT in @@ -493,6 +495,7 @@ class Command(NoArgsCommand): len(connection.queries) - queries_before, len(all_del_pks)) + @transaction.atomic def _delete_group_children_and_hosts(self): ''' Clear all invalid child relationships for groups and all invalid host @@ -554,6 +557,7 @@ class Command(NoArgsCommand): len(connection.queries) - queries_before, group_group_count + group_host_count) + @transaction.atomic def _update_inventory(self): ''' Update/overwrite variables from "all" group. If importing from a @@ -578,6 +582,7 @@ class Command(NoArgsCommand): else: logger.info('%s variables unmodified', all_name.capitalize()) + @transaction.atomic def _create_update_groups(self): ''' For each group in the local list, create it if it doesn't exist in the @@ -619,7 +624,7 @@ class Command(NoArgsCommand): if group_name in existing_group_names: continue mem_group = self.all_group.all_groups[group_name] - group = self.inventory.groups.create(name=group_name, variables=json.dumps(mem_group.variables), description='imported') + group, createdGroup = self.inventory.groups.get_or_create(name=group_name, variables=json.dumps(mem_group.variables), description='imported') logger.info('Group "%s" added', group.name) self._batch_add_m2m(self.inventory_source.groups, group) self._batch_add_m2m(self.inventory_source.groups, flush=True) @@ -628,6 +633,7 @@ class Command(NoArgsCommand): len(connection.queries) - queries_before, len(self.all_group.all_groups)) + @transaction.atomic def _update_db_host_from_mem_host(self, db_host, mem_host): # Update host variables. db_variables = db_host.variables_dict @@ -763,6 +769,7 @@ class Command(NoArgsCommand): len(connection.queries) - queries_before, len(self.all_group.all_hosts)) + @transaction.atomic def _create_update_group_children(self): ''' For each imported group, create all parent-child group relationships. @@ -790,6 +797,7 @@ class Command(NoArgsCommand): logger.warning('Group-group updates took %d queries for %d group-group relationships', len(connection.queries) - queries_before, group_group_count) + @transaction.atomic def _create_update_group_hosts(self): # For each host in a mem group, add it to the parent(s) to which it # belongs. From 5a307dd62b2bf6e5719c9de3768214a13b3cdaaf Mon Sep 17 00:00:00 2001 From: adamscmRH Date: Wed, 12 Jul 2017 22:34:15 -0400 Subject: [PATCH 3/5] Added defaults to get_or_create methods --- awx/main/management/commands/inventory_import.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 984ccb11d2..575fb8d714 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -624,7 +624,7 @@ class Command(NoArgsCommand): if group_name in existing_group_names: continue mem_group = self.all_group.all_groups[group_name] - group, createdGroup = self.inventory.groups.get_or_create(name=group_name, variables=json.dumps(mem_group.variables), description='imported') + group = self.inventory.groups.get_or_create(name=group_name, defaults={'variables':json.dumps(mem_group.variables), 'description':'imported'})[0] logger.info('Group "%s" added', group.name) self._batch_add_m2m(self.inventory_source.groups, group) self._batch_add_m2m(self.inventory_source.groups, flush=True) @@ -748,14 +748,14 @@ class Command(NoArgsCommand): for mem_host_name in sorted(mem_host_names_to_update): mem_host = self.all_group.all_hosts[mem_host_name] host_attrs = dict(variables=json.dumps(mem_host.variables), - name=mem_host_name, description='imported') + description='imported') enabled = self._get_enabled(mem_host.variables) if enabled is not None: host_attrs['enabled'] = enabled if self.instance_id_var: instance_id = self._get_instance_id(mem_host.variables) host_attrs['instance_id'] = instance_id - db_host, createdHost = self.inventory.hosts.get_or_create(**host_attrs) + db_host = self.inventory.hosts.get_or_create(name=mem_host_name, defaults={'variables':json.dumps(host_attrs)})[0] if enabled is False: logger.info('Host "%s" added (disabled)', mem_host_name) else: From 0d0c75c3180633c8ad6c29a1540a03959a9e501e Mon Sep 17 00:00:00 2001 From: adamscmRH Date: Thu, 13 Jul 2017 13:55:42 -0400 Subject: [PATCH 4/5] Change to update if already exists --- awx/main/management/commands/inventory_import.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 575fb8d714..42612e185e 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -624,7 +624,7 @@ class Command(NoArgsCommand): if group_name in existing_group_names: continue mem_group = self.all_group.all_groups[group_name] - group = self.inventory.groups.get_or_create(name=group_name, defaults={'variables':json.dumps(mem_group.variables), 'description':'imported'})[0] + group = self.inventory.groups.update_or_create(name=group_name, defaults={'variables':json.dumps(mem_group.variables), 'description':'imported'})[0] logger.info('Group "%s" added', group.name) self._batch_add_m2m(self.inventory_source.groups, group) self._batch_add_m2m(self.inventory_source.groups, flush=True) @@ -755,7 +755,8 @@ class Command(NoArgsCommand): if self.instance_id_var: instance_id = self._get_instance_id(mem_host.variables) host_attrs['instance_id'] = instance_id - db_host = self.inventory.hosts.get_or_create(name=mem_host_name, defaults={'variables':json.dumps(host_attrs)})[0] + db_host = self.inventory.hosts.update_or_create(name=mem_host_name, + defaults={'variables':host_attrs['variables'], 'description':host_attrs['description']})[0] if enabled is False: logger.info('Host "%s" added (disabled)', mem_host_name) else: From d065d5176ee84a4d55966ed115f315ccd0b679c5 Mon Sep 17 00:00:00 2001 From: adamscmRH Date: Fri, 14 Jul 2017 06:41:53 -0700 Subject: [PATCH 5/5] Removes redundant atomic transactions --- awx/main/management/commands/inventory_import.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 42612e185e..64e997745b 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -421,7 +421,6 @@ class Command(NoArgsCommand): mem_host.instance_id = instance_id self.mem_instance_id_map[instance_id] = mem_host.name - @transaction.atomic def _delete_hosts(self): ''' For each host in the database that is NOT in the local list, delete @@ -463,7 +462,6 @@ class Command(NoArgsCommand): len(connection.queries) - queries_before, len(all_del_pks)) - @transaction.atomic def _delete_groups(self): ''' # If overwrite is set, for each group in the database that is NOT in @@ -495,7 +493,6 @@ class Command(NoArgsCommand): len(connection.queries) - queries_before, len(all_del_pks)) - @transaction.atomic def _delete_group_children_and_hosts(self): ''' Clear all invalid child relationships for groups and all invalid host @@ -557,7 +554,6 @@ class Command(NoArgsCommand): len(connection.queries) - queries_before, group_group_count + group_host_count) - @transaction.atomic def _update_inventory(self): ''' Update/overwrite variables from "all" group. If importing from a @@ -582,7 +578,6 @@ class Command(NoArgsCommand): else: logger.info('%s variables unmodified', all_name.capitalize()) - @transaction.atomic def _create_update_groups(self): ''' For each group in the local list, create it if it doesn't exist in the @@ -633,7 +628,6 @@ class Command(NoArgsCommand): len(connection.queries) - queries_before, len(self.all_group.all_groups)) - @transaction.atomic def _update_db_host_from_mem_host(self, db_host, mem_host): # Update host variables. db_variables = db_host.variables_dict @@ -685,7 +679,6 @@ class Command(NoArgsCommand): logger.info('Host "%s" is now disabled', mem_host.name) self._batch_add_m2m(self.inventory_source.hosts, db_host) - @transaction.atomic def _create_update_hosts(self): ''' For each host in the local list, create it if it doesn't exist in the