Make cloud providers dynamic (#15537)

* Add dynamic pull for cloud inventory plugins and update corresponding tests

Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua>

* Create third dictionary to preserve current functionality and add 'file' there

* Migrations for corresponding change

---------

Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua>

awx/api/serializers.py

@@ -102,7 +102,6 @@ from awx.main.models import (
     WorkflowJobTemplate,
     WorkflowJobTemplateNode,
     StdoutMaxBytesExceeded,
-    CLOUD_INVENTORY_SOURCES,
 )
 from awx.main.models.base import VERBOSITY_CHOICES, NEW_JOB_TYPE_CHOICES
 from awx.main.models.rbac import role_summary_fields_generator, give_creator_permissions, get_role_codenames, to_permissions, get_role_from_object_role
@@ -119,7 +118,9 @@ from awx.main.utils import (
     truncate_stdout,
     get_licenser,
 )
+
 from awx.main.utils.filters import SmartFilter
+from awx.main.utils.plugins import load_combined_inventory_source_options
 from awx.main.utils.named_url_graph import reset_counters
 from awx.main.scheduler.task_manager_models import TaskManagerModels
 from awx.main.redact import UriCleaner, REPLACE_STR
@@ -2300,6 +2301,7 @@ class GroupVariableDataSerializer(BaseVariableDataSerializer):
 
 class InventorySourceOptionsSerializer(BaseSerializer):
     credential = DeprecatedCredentialField(help_text=_('Cloud credential to use for inventory updates.'))
+    source = serializers.ChoiceField(choices=[])
 
     class Meta:
         fields = (
@@ -2321,6 +2323,11 @@ class InventorySourceOptionsSerializer(BaseSerializer):
         )
         read_only_fields = ('*', 'custom_virtualenv')
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if 'source' in self.fields:
+            self.fields['source'].choices = load_combined_inventory_source_options()
+
     def get_related(self, obj):
         res = super(InventorySourceOptionsSerializer, self).get_related(obj)
         if obj.credential:  # TODO: remove when 'credential' field is removed
@@ -5500,7 +5507,7 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
         return summary_fields
 
     def validate_unified_job_template(self, value):
-        if type(value) == InventorySource and value.source not in CLOUD_INVENTORY_SOURCES:
+        if type(value) == InventorySource and value.source not in load_combined_inventory_source_options():
             raise serializers.ValidationError(_('Inventory Source must be a cloud resource.'))
         elif type(value) == Project and value.scm_type == '':
             raise serializers.ValidationError(_('Manual Project cannot have a schedule set.'))

awx/api/views/__init__.py

@@ -100,6 +100,7 @@ from awx.main.utils import (
 )
 from awx.main.utils.encryption import encrypt_value
 from awx.main.utils.filters import SmartFilter
+from awx.main.utils.plugins import compute_cloud_inventory_sources
 from awx.main.redact import UriCleaner
 from awx.api.permissions import (
     JobTemplateCallbackPermission,
@@ -2196,9 +2197,9 @@ class InventorySourceNotificationTemplatesAnyList(SubListCreateAttachDetachAPIVi
 
     def post(self, request, *args, **kwargs):
         parent = self.get_parent_object()
-        if parent.source not in models.CLOUD_INVENTORY_SOURCES:
+        if parent.source not in compute_cloud_inventory_sources():
             return Response(
-                dict(msg=_("Notification Templates can only be assigned when source is one of {}.").format(models.CLOUD_INVENTORY_SOURCES, parent.source)),
+                dict(msg=_("Notification Templates can only be assigned when source is one of {}.").format(compute_cloud_inventory_sources(), parent.source)),
                 status=status.HTTP_400_BAD_REQUEST,
             )
         return super(InventorySourceNotificationTemplatesAnyList, self).post(request, *args, **kwargs)

awx/main/constants.py

@@ -6,7 +6,6 @@ import re
 from django.utils.translation import gettext_lazy as _
 
 __all__ = [
-    'CLOUD_PROVIDERS',
     'PRIVILEGE_ESCALATION_METHODS',
     'ANSI_SGR_PATTERN',
     'CAN_CANCEL',
@@ -14,25 +13,6 @@ __all__ = [
     'STANDARD_INVENTORY_UPDATE_ENV',
 ]
 
-CLOUD_PROVIDERS = (
-    'azure_rm',
-    'ec2',
-    'gce',
-    'vmware',
-    'openstack',
-    'rhv',
-    'satellite6',
-    'controller',
-    'insights',
-    'terraform',
-    'openshift_virtualization',
-    'controller_supported',
-    'rhv_supported',
-    'openshift_virtualization_supported',
-    'insights_supported',
-    'satellite6_supported',
-)
-
 PRIVILEGE_ESCALATION_METHODS = [
     ('sudo', _('Sudo')),
     ('su', _('Su')),

awx/main/migrations/0198_alter_inventorysource_source_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.10 on 2024-10-22 15:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0197_remove_sso_app_content'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='inventorysource',
+            name='source',
+            field=models.CharField(default=None, max_length=32),
+        ),
+        migrations.AlterField(
+            model_name='inventoryupdate',
+            name='source',
+            field=models.CharField(default=None, max_length=32),
+        ),
+    ]

awx/main/models/__init__.py

@@ -16,7 +16,7 @@ from ansible_base.lib.utils.models import prevent_search
 from ansible_base.lib.utils.models import user_summary_fields
 
 # AWX
-from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES  # noqa
+from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, VERBOSITY_CHOICES  # noqa
 from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded  # noqa
 from awx.main.models.organization import Organization, Team, UserSessionMembership  # noqa
 from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env  # noqa

awx/main/models/base.py

@@ -15,7 +15,6 @@ from crum import get_current_user
 
 # AWX
 from awx.main.utils import encrypt_field, parse_yaml_or_json
-from awx.main.constants import CLOUD_PROVIDERS
 
 __all__ = [
     'VarsDictProperty',
@@ -32,7 +31,6 @@ __all__ = [
     'JOB_TYPE_CHOICES',
     'AD_HOC_JOB_TYPE_CHOICES',
     'PROJECT_UPDATE_JOB_TYPE_CHOICES',
-    'CLOUD_INVENTORY_SOURCES',
     'VERBOSITY_CHOICES',
 ]
 
@@ -61,7 +59,6 @@ PROJECT_UPDATE_JOB_TYPE_CHOICES = [
     (PERM_INVENTORY_CHECK, _('Check')),
 ]
 
-CLOUD_INVENTORY_SOURCES = list(CLOUD_PROVIDERS) + ['scm']
 
 VERBOSITY_CHOICES = [
     (0, '0 (Normal)'),

awx/main/models/inventory.py

@@ -28,7 +28,7 @@ from awx_plugins.inventory.plugins import PluginFileInjector
 
 # AWX
 from awx.api.versioning import reverse
-from awx.main.constants import CLOUD_PROVIDERS
+from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names, compute_cloud_inventory_sources
 from awx.main.consumers import emit_channel_notification
 from awx.main.fields import (
     ImplicitRoleField,
@@ -36,7 +36,7 @@ from awx.main.fields import (
     OrderedManyToManyField,
 )
 from awx.main.managers import HostManager, HostMetricActiveManager
-from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, accepts_json
+from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, accepts_json
 from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent
 from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
 from awx.main.models.mixins import (
@@ -394,7 +394,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
         if self.kind == 'smart':
             active_inventory_sources = self.inventory_sources.none()
         else:
-            active_inventory_sources = self.inventory_sources.filter(source__in=CLOUD_INVENTORY_SOURCES)
+            active_inventory_sources = self.inventory_sources.filter(source__in=compute_cloud_inventory_sources())
         failed_inventory_sources = active_inventory_sources.filter(last_job_failed=True)
         total_hosts = active_hosts.count()
         # if total_hosts has changed, set update_task_impact to True
@@ -914,23 +914,6 @@ class InventorySourceOptions(BaseModel):
 
     injectors = dict()
 
-    SOURCE_CHOICES = [
-        ('file', _('File, Directory or Script')),
-        ('constructed', _('Template additional groups and hostvars at runtime')),
-        ('scm', _('Sourced from a Project')),
-        ('ec2', _('Amazon EC2')),
-        ('gce', _('Google Compute Engine')),
-        ('azure_rm', _('Microsoft Azure Resource Manager')),
-        ('vmware', _('VMware vCenter')),
-        ('satellite6', _('Red Hat Satellite 6')),
-        ('openstack', _('OpenStack')),
-        ('rhv', _('Red Hat Virtualization')),
-        ('controller', _('Red Hat Ansible Automation Platform')),
-        ('insights', _('Red Hat Insights')),
-        ('terraform', _('Terraform State')),
-        ('openshift_virtualization', _('OpenShift Virtualization')),
-    ]
-
     # From the options of the Django management base command
     INVENTORY_UPDATE_VERBOSITY_CHOICES = [
         (0, '0 (WARNING)'),
@@ -943,7 +926,6 @@ class InventorySourceOptions(BaseModel):
 
     source = models.CharField(
         max_length=32,
-        choices=SOURCE_CHOICES,
         blank=False,
         default=None,
     )
@@ -1047,7 +1029,7 @@ class InventorySourceOptions(BaseModel):
         # Allow an EC2 source to omit the credential.  If Tower is running on
         # an EC2 instance with an IAM Role assigned, boto will use credentials
         # from the instance metadata instead of those explicitly provided.
-        elif source in CLOUD_PROVIDERS and source not in ['ec2', 'openshift_virtualization']:
+        elif source in discover_available_cloud_provider_plugin_names() and source not in ['ec2', 'openshift_virtualization']:
             return _('Credential is required for a cloud source.')
         elif source == 'custom' and cred and cred.credential_type.kind in ('scm', 'ssh', 'insights', 'vault'):
             return _('Credentials of type machine, source control, insights and vault are disallowed for custom inventory sources.')
@@ -1061,11 +1043,8 @@ class InventorySourceOptions(BaseModel):
         """Return the credential which is directly tied to the inventory source type."""
         credential = None
         for cred in self.credentials.all():
-            if self.source in CLOUD_PROVIDERS:
+            if self.source in discover_available_cloud_provider_plugin_names():
-                source = self.source.replace('ec2', 'aws')
+                if cred.kind == self.source.replace('ec2', 'aws'):
-                if source.endswith('_supported'):
-                    source = source[:-10]
-                if cred.kind == source:
                     credential = cred
                     break
             else:
@@ -1080,7 +1059,7 @@ class InventorySourceOptions(BaseModel):
         These are all credentials that should run their own inject_credential logic.
         """
         special_cred = None
-        if self.source in CLOUD_PROVIDERS:
+        if self.source in discover_available_cloud_provider_plugin_names():
             # these have special injection logic associated with them
             special_cred = self.get_cloud_credential()
         extra_creds = []

awx/main/tests/functional/models/test_inventory.py

@@ -5,8 +5,8 @@ from unittest import mock
 
 # AWX
 from awx.main.models import Host, Inventory, InventorySource, InventoryUpdate, CredentialType, Credential, Job
-from awx.main.constants import CLOUD_PROVIDERS
 from awx.main.utils.filters import SmartFilter
+from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names
 
 
 @pytest.mark.django_db
@@ -166,11 +166,11 @@ class TestInventorySourceInjectors:
 
     def test_all_cloud_sources_covered(self):
         """Code in several places relies on the fact that the older
-        CLOUD_PROVIDERS constant contains the same names as what are
+        discover_cloud_provider_plugin_names returns the same names as what are
         defined within the injectors
         """
         # slight exception case for constructed, because it has a FQCN but is not a cloud source
-        assert set(CLOUD_PROVIDERS) | set(['constructed']) == set(InventorySource.injectors.keys())
+        assert set(discover_available_cloud_provider_plugin_names()) | set(['constructed']) == set(InventorySource.injectors.keys())
 
     @pytest.mark.parametrize('source,filename', [('ec2', 'aws_ec2.yml'), ('openstack', 'openstack.yml'), ('gce', 'gcp_compute.yml')])
     def test_plugin_filenames(self, source, filename):

awx/main/tests/functional/test_inventory_source_injectors.py

@@ -9,9 +9,9 @@ from awx_plugins.interfaces._temporary_private_container_api import get_incontai
 
 from awx.main.tasks.jobs import RunInventoryUpdate
 from awx.main.models import InventorySource, Credential, CredentialType, UnifiedJob, ExecutionEnvironment
-from awx.main.constants import CLOUD_PROVIDERS, STANDARD_INVENTORY_UPDATE_ENV
+from awx.main.constants import STANDARD_INVENTORY_UPDATE_ENV
 from awx.main.tests import data
-
+from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names
 from django.conf import settings
 
 DATA = os.path.join(os.path.dirname(data.__file__), 'inventory')
@@ -193,7 +193,7 @@ def create_reference_data(source_dir, env, content):
 
 
 @pytest.mark.django_db
-@pytest.mark.parametrize('this_kind', CLOUD_PROVIDERS)
+@pytest.mark.parametrize('this_kind', discover_available_cloud_provider_plugin_names())
 def test_inventory_update_injected_content(this_kind, inventory, fake_credential_factory, mock_me):
     if this_kind.endswith('_supported'):
         this_kind = this_kind[:-10]
@@ -202,8 +202,6 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential
     ExecutionEnvironment.objects.create(name='Default Job EE', managed=False)
 
     injector = InventorySource.injectors[this_kind]
-    if injector.plugin_name is None:
-        pytest.skip('Use of inventory plugin is not enabled for this source')
 
     src_vars = dict(base_source_var='value_of_var')
     src_vars['plugin'] = injector.get_proper_name()

awx/main/utils/plugins.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2024 Ansible, Inc.
+# All Rights Reserved.
+
+"""
+This module contains the code responsible for extracting the lists of dynamically discovered plugins.
+"""
+
+from functools import cache
+
+
+@cache
+def discover_available_cloud_provider_plugin_names() -> list[str]:
+    """
+    Return a list of cloud plugin names available in runtime.
+
+    The discovery result is cached since it does not change throughout
+    the life cycle of the server run.
+
+    :returns: List of plugin cloud names.
+    :rtype: list[str]
+    """
+    from awx.main.models.inventory import InventorySourceOptions
+
+    plugin_names = list(InventorySourceOptions.injectors.keys())
+
+    plugin_names.remove('constructed')
+
+    return plugin_names
+
+
+@cache
+def compute_cloud_inventory_sources() -> dict[str, str]:
+    """
+    Return a dictionary of cloud provider plugin names
+    available plus source control management and constructed.
+
+    :returns: Dictionary of plugin cloud names plus source control.
+    :rtype: dict[str, str]
+    """
+
+    plugins = discover_available_cloud_provider_plugin_names()
+
+    return dict(zip(plugins, plugins), scm='scm', constructed='constructed')
+
+
+@cache
+def load_combined_inventory_source_options() -> dict[str, str]:
+    """
+    Return a dictionary of cloud provider plugin names and 'file'.
+
+    The 'file' entry is included separately since it needs to be consumed directly by the serializer.
+
+    :returns: A dictionary of cloud provider plugin names (as both keys and values) plus the 'file' entry.
+    :rtype: dict[str, str]
+    """
+
+    plugins = compute_cloud_inventory_sources()
+
+    return dict(zip(plugins, plugins), file='file')