Merge pull request #6553 from ryanpetrello/remove-manual-inv-source-for-good

remove deprecated manual inventory source support

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

awx/api/serializers.py

@@ -2034,11 +2034,6 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
             res['credentials'] = self.reverse('api:inventory_source_credentials_list', kwargs={'pk': obj.pk})
         return res
 
-    def get_group(self, obj):  # TODO: remove in 3.3
-        if obj.deprecated_group:
-            return obj.deprecated_group.id
-        return None
-
     def build_relational_field(self, field_name, relation_info):
         field_class, field_kwargs = super(InventorySourceSerializer, self).build_relational_field(field_name, relation_info)
         # SCM Project and inventory are read-only unless creating a new inventory.

awx/main/access.py

@@ -405,14 +405,6 @@ class BaseAccess(object):
                 # Cannot copy manual project without errors
                 user_capabilities[display_method] = False
                 continue
-            elif display_method in ['start', 'schedule'] and isinstance(obj, Group):  # TODO: remove in 3.3
-                try:
-                    if obj.deprecated_inventory_source and not obj.deprecated_inventory_source._can_update():
-                        user_capabilities[display_method] = False
-                        continue
-                except Group.deprecated_inventory_source.RelatedObjectDoesNotExist:
-                    user_capabilities[display_method] = False
-                    continue
             elif display_method in ['start', 'schedule'] and isinstance(obj, (Project)):
                 if obj.scm_type == '':
                     user_capabilities[display_method] = False

awx/main/management/commands/inventory_import.py

@@ -496,12 +496,6 @@ class Command(BaseCommand):
             group_names = all_group_names[offset:(offset + self._batch_size)]
             for group_pk in groups_qs.filter(name__in=group_names).values_list('pk', flat=True):
                 del_group_pks.discard(group_pk)
-        if self.inventory_source.deprecated_group_id in del_group_pks:  # TODO: remove in 3.3
-            logger.warning(
-                'Group "%s" from v1 API is not deleted by overwrite',
-                self.inventory_source.deprecated_group.name
-            )
-            del_group_pks.discard(self.inventory_source.deprecated_group_id)
         # Now delete all remaining groups in batches.
         all_del_pks = sorted(list(del_group_pks))
         for offset in range(0, len(all_del_pks), self._batch_size):
@@ -534,12 +528,6 @@ class Command(BaseCommand):
         # Set of all host pks managed by this inventory source
         all_source_host_pks = self._existing_host_pks()
         for db_group in db_groups.all():
-            if self.inventory_source.deprecated_group_id == db_group.id:  # TODO: remove in 3.3
-                logger.debug(
-                    'Group "%s" from v1 API child group/host connections preserved',
-                    db_group.name
-                )
-                continue
             # Delete child group relationships not present in imported data.
             db_children = db_group.children
             db_children_name_pk_map = dict(db_children.values_list('name', 'pk'))

awx/main/migrations/0114_v370_remove_deprecated_manual_inventory_sources.py

@@ -0,0 +1,39 @@
+# Generated by Django 2.2.11 on 2020-04-03 00:11
+
+from django.db import migrations, models
+
+
+def remove_manual_inventory_sources(apps, schema_editor):
+    '''Previously we would automatically create inventory sources after
+    Group creation and we would use the parent Group as our interface for the user.
+    During that process we would create InventorySource that had a source of "manual".
+    '''
+    InventoryUpdate = apps.get_model('main', 'InventoryUpdate')
+    InventoryUpdate.objects.filter(source='').delete()
+    InventorySource = apps.get_model('main', 'InventorySource')
+    InventorySource.objects.filter(source='').delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0113_v370_event_bigint'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='inventorysource',
+            name='deprecated_group',
+        ),
+        migrations.RunPython(remove_manual_inventory_sources),
+        migrations.AlterField(
+            model_name='inventorysource',
+            name='source',
+            field=models.CharField(choices=[('file', 'File, Directory or Script'), ('scm', 'Sourced from a Project'), ('ec2', 'Amazon EC2'), ('gce', 'Google Compute Engine'), ('azure_rm', 'Microsoft Azure Resource Manager'), ('vmware', 'VMware vCenter'), ('satellite6', 'Red Hat Satellite 6'), ('cloudforms', 'Red Hat CloudForms'), ('openstack', 'OpenStack'), ('rhv', 'Red Hat Virtualization'), ('tower', 'Ansible Tower'), ('custom', 'Custom Script')], default=None, max_length=32),
+        ),
+        migrations.AlterField(
+            model_name='inventoryupdate',
+            name='source',
+            field=models.CharField(choices=[('file', 'File, Directory or Script'), ('scm', 'Sourced from a Project'), ('ec2', 'Amazon EC2'), ('gce', 'Google Compute Engine'), ('azure_rm', 'Microsoft Azure Resource Manager'), ('vmware', 'VMware vCenter'), ('satellite6', 'Red Hat Satellite 6'), ('cloudforms', 'Red Hat CloudForms'), ('openstack', 'OpenStack'), ('rhv', 'Red Hat Virtualization'), ('tower', 'Ansible Tower'), ('custom', 'Custom Script')], default=None, max_length=32),
+        ),
+    ]

awx/main/models/inventory.py

@@ -821,7 +821,6 @@ class InventorySourceOptions(BaseModel):
     injectors = dict()
 
     SOURCE_CHOICES = [
-        ('', _('Manual')),
         ('file', _('File, Directory or Script')),
         ('scm', _('Sourced from a Project')),
         ('ec2', _('Amazon EC2')),
@@ -932,8 +931,8 @@ class InventorySourceOptions(BaseModel):
     source = models.CharField(
         max_length=32,
         choices=SOURCE_CHOICES,
-        blank=True,
-        default='',
+        blank=False,
+        default=None,
     )
     source_path = models.CharField(
         max_length=1024,
@@ -1237,14 +1236,6 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
         on_delete=models.CASCADE,
     )
 
-    deprecated_group = models.OneToOneField(
-        'Group',
-        related_name='deprecated_inventory_source',
-        null=True,
-        default=None,
-        on_delete=models.CASCADE,
-    )
-
     source_project = models.ForeignKey(
         'Project',
         related_name='scm_inventory_sources',
@@ -1345,12 +1336,6 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
     def get_absolute_url(self, request=None):
         return reverse('api:inventory_source_detail', kwargs={'pk': self.pk}, request=request)
 
-    @property
-    def can_update(self):
-        if self.source == '':
-            return False
-        return super(InventorySource, self).can_update
-
     def _can_update(self):
         if self.source == 'custom':
             return bool(self.source_script)
@@ -1420,16 +1405,6 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
                     started=list(started_notification_templates),
                     success=list(success_notification_templates))
 
-    def clean_source(self):  # TODO: remove in 3.3
-        source = self.source
-        if source and self.deprecated_group:
-            qs = self.deprecated_group.inventory_sources.filter(source__in=CLOUD_INVENTORY_SOURCES)
-            existing_sources = qs.exclude(pk=self.pk)
-            if existing_sources.count():
-                s = u', '.join([x.deprecated_group.name for x in existing_sources])
-                raise ValidationError(_('Unable to configure this item for cloud sync. It is already managed by %s.') % s)
-        return source
-
     def clean_update_on_project_update(self):
         if self.update_on_project_update is True and \
                 self.source == 'scm' and \
@@ -1518,8 +1493,6 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
         if self.inventory_source.inventory is not None:
             websocket_data.update(dict(inventory_id=self.inventory_source.inventory.pk))
 
-        if self.inventory_source.deprecated_group is not None:  # TODO: remove in 3.3
-            websocket_data.update(dict(group_id=self.inventory_source.deprecated_group.id))
         return websocket_data
 
     def get_absolute_url(self, request=None):

awx/main/signals.py

@@ -382,10 +382,6 @@ def emit_activity_stream_change(instance):
 
 def activity_stream_create(sender, instance, created, **kwargs):
     if created and activity_stream_enabled:
-        # TODO: remove deprecated_group conditional in 3.3
-        # Skip recording any inventory source directly associated with a group.
-        if isinstance(instance, InventorySource) and instance.deprecated_group:
-            return
         _type = type(instance)
         if getattr(_type, '_deferred', False):
             return
@@ -458,10 +454,6 @@ def activity_stream_update(sender, instance, **kwargs):
 def activity_stream_delete(sender, instance, **kwargs):
     if not activity_stream_enabled:
         return
-    # TODO: remove deprecated_group conditional in 3.3
-    # Skip recording any inventory source directly associated with a group.
-    if isinstance(instance, InventorySource) and instance.deprecated_group:
-        return
     # Inventory delete happens in the task system rather than request-response-cycle.
     # If we trigger this handler there we may fall into db-integrity-related race conditions.
     # So we add flag verification to prevent normal signal handling. This funciton will be

awx/main/tests/functional/api/test_credential.py

@@ -972,7 +972,7 @@ def test_field_removal(put, organization, admin, credentialtype_ssh):
     ['insights_inventories', Inventory()],
     ['unifiedjobs', Job()],
     ['unifiedjobtemplates', JobTemplate()],
-    ['unifiedjobtemplates', InventorySource()],
+    ['unifiedjobtemplates', InventorySource(source='ec2')],
     ['projects', Project()],
     ['workflowjobnodes', WorkflowJobNode()],
 ])

awx/main/tests/functional/api/test_unified_jobs_stdout.py

@@ -23,9 +23,9 @@ def _mk_project_update():
 
 
 def _mk_inventory_update():
-    source = InventorySource()
+    source = InventorySource(source='ec2')
     source.save()
-    iu = InventoryUpdate(inventory_source=source)
+    iu = InventoryUpdate(inventory_source=source, source='e2')
     return iu
 
 

awx/main/tests/functional/api/test_unified_jobs_view.py

@@ -123,7 +123,11 @@ def test_delete_project_update_in_active_state(project, delete, admin, status):
 @pytest.mark.parametrize("status", list(TEST_STATES))
 @pytest.mark.django_db
 def test_delete_inventory_update_in_active_state(inventory_source, delete, admin, status):
-    i = InventoryUpdate.objects.create(inventory_source=inventory_source, status=status)
+    i = InventoryUpdate.objects.create(
+        inventory_source=inventory_source,
+        status=status,
+        source=inventory_source.source
+    )
     url = reverse('api:inventory_update_detail', kwargs={'pk': i.pk})
     delete(url, None, admin, expect=403)
 

awx/main/tests/functional/commands/test_inventory_import.py

@@ -228,7 +228,7 @@ class TestINIImports:
         assert inventory.hosts.count() == 1  # baseline worked
 
         inv_src2 = inventory.inventory_sources.create(
-            name='bar', overwrite=True
+            name='bar', overwrite=True, source='ec2'
         )
         os.environ['INVENTORY_SOURCE_ID'] = str(inv_src2.pk)
         os.environ['INVENTORY_UPDATE_ID'] = str(inv_src2.create_unified_job().pk)

awx/main/tests/functional/conftest.py

@@ -568,7 +568,10 @@ def inventory_source_factory(inventory_factory):
 
 @pytest.fixture
 def inventory_update(inventory_source):
-    return InventoryUpdate.objects.create(inventory_source=inventory_source)
+    return InventoryUpdate.objects.create(
+        inventory_source=inventory_source,
+        source=inventory_source.source
+    )
 
 
 @pytest.fixture

awx/main/tests/functional/models/test_inventory.py

@@ -197,9 +197,10 @@ class TestRelatedJobs:
         assert job.id in [jerb.id for jerb in group._get_related_jobs()]
 
     def test_related_group_update(self, group):
-        src = group.inventory_sources.create(name='foo')
+        src = group.inventory_sources.create(name='foo', source='ec2')
         job = InventoryUpdate.objects.create(
-            inventory_source=src
+            inventory_source=src,
+            source=src.source
         )
         assert job.id in [jerb.id for jerb in group._get_related_jobs()]
 

awx/main/tests/functional/models/test_notifications.py

@@ -109,6 +109,7 @@ class TestJobNotificationMixin(object):
         kwargs = {}
         if JobClass is InventoryUpdate:
             kwargs['inventory_source'] = inventory_source
+            kwargs['source'] = inventory_source.source
         elif JobClass is ProjectUpdate:
             kwargs['project'] = project
 

awx/main/tests/functional/test_instances.py

@@ -297,7 +297,10 @@ class TestInstanceGroupOrdering:
         assert ad_hoc.preferred_instance_groups == [ig_inv, ig_org]
 
     def test_inventory_update_instance_groups(self, instance_group_factory, inventory_source, default_instance_group):
-        iu = InventoryUpdate.objects.create(inventory_source=inventory_source)
+        iu = InventoryUpdate.objects.create(
+            inventory_source=inventory_source,
+            source=inventory_source.source
+        )
         assert iu.preferred_instance_groups == [default_instance_group]
         ig_org = instance_group_factory("OrgIstGrp", [default_instance_group.instances.first()])
         ig_inv = instance_group_factory("InvIstGrp", [default_instance_group.instances.first()])

awx/main/tests/functional/test_named_url.py

@@ -186,7 +186,11 @@ def test_group(get, admin_user):
 def test_inventory_source(get, admin_user):
     test_org = Organization.objects.create(name='test_org')
     test_inv = Inventory.objects.create(name='test_inv', organization=test_org)
-    test_source = InventorySource.objects.create(name='test_source', inventory=test_inv)
+    test_source = InventorySource.objects.create(
+        name='test_source',
+        inventory=test_inv,
+        source='ec2'
+    )
     url = reverse('api:inventory_source_detail', kwargs={'pk': test_source.pk})
     response = get(url, user=admin_user, expect=200)
     assert response.data['related']['named_url'].endswith('/test_source++test_inv++test_org/')

awx/main/tests/functional/test_notifications.py

@@ -90,7 +90,7 @@ def test_inherited_notification_templates(get, post, user, organization, project
         notification_templates.append(response.data['id'])
     i = Inventory.objects.create(name='test', organization=organization)
     i.save()
-    isrc = InventorySource.objects.create(name='test', inventory=i)
+    isrc = InventorySource.objects.create(name='test', inventory=i, source='ec2')
     isrc.save()
     jt = JobTemplate.objects.create(name='test', inventory=i, project=project, playbook='debug.yml')
     jt.save()

awx/main/tests/functional/test_rbac_migration.py

@@ -24,7 +24,11 @@ def test_implied_organization_subquery_inventory():
             inventory = Inventory.objects.create(name='foo{}'.format(i))
         else:
             inventory = Inventory.objects.create(name='foo{}'.format(i), organization=org)
-        inv_src = InventorySource.objects.create(name='foo{}'.format(i), inventory=inventory)
+        inv_src = InventorySource.objects.create(
+            name='foo{}'.format(i),
+            inventory=inventory,
+            source='ec2'
+        )
     sources = UnifiedJobTemplate.objects.annotate(
         test_field=rbac.implicit_org_subquery(UnifiedJobTemplate, InventorySource)
     )

awx_collection/test/awx/test_workflow_template.py

@@ -68,7 +68,8 @@ def test_schema_with_branches(run_module, admin_user, organization, silence_depr
     )
     inv_src = InventorySource.objects.create(
         inventory=inv,
-        name='AWS servers'
+        name='AWS servers',
+        source='ec2'
     )
 
     result = run_module('tower_workflow_template', {