Merge pull request #6089 from ryanpetrello/credential_type_injectors

Implement custom credential type injectors

awx/main/fields.py

@@ -699,24 +699,6 @@ class CredentialTypeInjectorField(JSONSchemaField):
                     'additionalProperties': False,
                     'required': ['template'],
                 },
-                'ssh': {
-                    'type': 'object',
-                    'properties': {
-                        'private': {'type': 'string'},
-                        'public': {'type': 'string'},
-                    },
-                    'additionalProperties': False,
-                    'required': ['public', 'private'],
-                },
-                'password': {
-                    'type': 'object',
-                    'properties': {
-                        'key': {'type': 'string'},
-                        'value': {'type': 'string'},
-                    },
-                    'additionalProperties': False,
-                    'required': ['key', 'value'],
-                },
                 'env': {
                     'type': 'object',
                     'patternProperties': {

awx/main/models/credential.py

@@ -2,7 +2,14 @@
 # All Rights Reserved.
 from collections import OrderedDict
 import functools
+import json
 import operator
+import os
+import stat
+import tempfile
+
+# Jinja2
+from jinja2 import Template
 
 # Django
 from django.db import models
@@ -450,6 +457,14 @@ class CredentialType(CommonModelNameNotUnique):
 
     defaults = OrderedDict()
 
+    ENV_BLACKLIST = set((
+        'VIRTUAL_ENV', 'PATH', 'PYTHONPATH', 'PROOT_TMP_DIR', 'JOB_ID',
+        'INVENTORY_ID', 'INVENTORY_SOURCE_ID', 'INVENTORY_UPDATE_ID',
+        'AD_HOC_COMMAND_ID', 'REST_API_URL', 'REST_API_TOKEN', 'TOWER_HOST',
+        'MAX_EVENT_RES', 'CALLBACK_QUEUE', 'CALLBACK_CONNECTION', 'CACHE',
+        'JOB_CALLBACK_DEBUG', 'INVENTORY_HOSTVARS', 'FACT_QUEUE',
+    ))
+
     class Meta:
         app_label = 'main'
         ordering = ('kind', 'name')
@@ -539,6 +554,90 @@ class CredentialType(CommonModelNameNotUnique):
             match = cls.objects.filter(**requirements)[:1].get()
         return match
 
+    def inject_credential(self, credential, env, safe_env, args, safe_args, private_data_dir):
+        """
+        Inject credential data into the environment variables and arguments
+        passed to `ansible-playbook`
+
+        :param credential:       a :class:`awx.main.models.Credential` instance
+        :param env:              a dictionary of environment variables used in
+                                 the `ansible-playbook` call.  This method adds
+                                 additional environment variables based on
+                                 custom `env` injectors defined on this
+                                 CredentialType.
+        :param safe_env:         a dictionary of environment variables stored
+                                 in the database for the job run
+                                 (`UnifiedJob.job_env`); secret values should
+                                 be stripped
+        :param args:             a list of arguments passed to
+                                 `ansible-playbook` in the style of
+                                 `subprocess.call(args)`.  This method appends
+                                 additional arguments based on custom
+                                 `extra_vars` injectors defined on this
+                                 CredentialType.
+        :param safe_args:        a list of arguments stored in the database for
+                                 the job run (`UnifiedJob.job_args`); secret
+                                 values should be stripped
+        :param private_data_dir: a temporary directory to store files generated
+                                 by `file` injectors (like config files or key
+                                 files)
+        """
+        if not self.injectors:
+            return
+
+        class TowerNamespace:
+            filename = None
+
+        tower_namespace = TowerNamespace()
+
+        # maintain a normal namespace for building the ansible-playbook arguments (env and args)
+        namespace = {'tower': tower_namespace}
+
+        # maintain a sanitized namespace for building the DB-stored arguments (safe_env and safe_args)
+        safe_namespace = {'tower': tower_namespace}
+
+        # build a normal namespace with secret values decrypted (for
+        # ansible-playbook) and a safe namespace with secret values hidden (for
+        # DB storage)
+        for field_name, value in credential.inputs.items():
+            if field_name in self.secret_fields:
+                value = decrypt_field(credential, field_name)
+                safe_namespace[field_name] = '**********'
+            elif len(value):
+                safe_namespace[field_name] = value
+            if len(value):
+                namespace[field_name] = value
+
+        file_tmpl = self.injectors.get('file', {}).get('template')
+        if file_tmpl is not None:
+            # If a file template is provided, render the file and update the
+            # special `tower` template namespace so the filename can be
+            # referenced in other injectors
+            data = Template(file_tmpl).render(**namespace)
+            _, path = tempfile.mkstemp(dir=private_data_dir)
+            with open(path, 'w') as f:
+                f.write(data)
+            os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
+            namespace['tower'].filename = path
+
+        for env_var, tmpl in self.injectors.get('env', {}).items():
+            if env_var.startswith('ANSIBLE_') or env_var in self.ENV_BLACKLIST:
+                continue
+            env[env_var] = Template(tmpl).render(**namespace)
+            safe_env[env_var] = Template(tmpl).render(**safe_namespace)
+
+        extra_vars = {}
+        safe_extra_vars = {}
+        for var_name, tmpl in self.injectors.get('extra_vars', {}).items():
+            extra_vars[var_name] = Template(tmpl).render(**namespace)
+            safe_extra_vars[var_name] = Template(tmpl).render(**safe_namespace)
+
+        if extra_vars:
+            args.extend(['-e', json.dumps(extra_vars)])
+
+        if safe_extra_vars:
+            safe_args.extend(['-e', json.dumps(safe_extra_vars)])
+
 
 @CredentialType.default
 def ssh(cls):

awx/main/models/inventory.py

@@ -979,7 +979,7 @@ class InventorySourceOptions(BaseModel):
         if not self.source:
             return None
         cred = self.credential
-        if cred:
+        if cred and self.source != 'custom':
             # If a credential was provided, it's important that it matches
             # the actual inventory source being used (Amazon requires Amazon
             # credentials; Rackspace requires Rackspace credentials; etc...)

awx/main/tasks.py

@@ -705,6 +705,15 @@ class BaseTask(Task):
             cwd = self.build_cwd(instance, **kwargs)
             env = self.build_env(instance, **kwargs)
             safe_env = self.build_safe_env(env, **kwargs)
+
+            # handle custom injectors specified on the CredentialType
+            for type_ in ('credential', 'cloud_credential', 'network_credential'):
+                credential = getattr(instance, type_, None)
+                if credential:
+                    credential.credential_type.inject_credential(
+                        credential, env, safe_env, args, safe_args, kwargs['private_data_dir']
+                    )
+
             stdout_handle = self.get_stdout_handle(instance)
             if self.should_use_proot(instance, **kwargs):
                 if not check_proot_installed():

awx/main/tests/functional/test_credential.py

@@ -94,17 +94,6 @@ def test_cred_type_input_schema_validity(input_, valid):
     ({'file': {}}, False),
     ({'file': {'template': '{{username}}'}}, True),
     ({'file': {'foo': 'bar'}}, False),
-    ({'ssh': 123}, False),
-    ({'ssh': {}}, False),
-    ({'ssh': {'public': 'PUB'}}, False),
-    ({'ssh': {'private': 'PRIV'}}, False),
-    ({'ssh': {'public': 'PUB', 'private': 'PRIV'}}, True),
-    ({'ssh': {'public': 'PUB', 'private': 'PRIV', 'a': 'b'}}, False),
-    ({'password': {}}, False),
-    ({'password': {'key': 'Password:'}}, False),
-    ({'password': {'value': '{{pass}}'}}, False),
-    ({'password': {'key': 'Password:', 'value': '{{pass}}'}}, True),
-    ({'password': {'key': 'Password:', 'value': '{{pass}}', 'a': 'b'}}, False),
     ({'env': 123}, False),
     ({'env': {}}, True),
     ({'env': {'AWX_SECRET': '{{awx_secret}}'}}, True),

awx/main/tests/unit/test_tasks.py

@@ -2,6 +2,7 @@ from contextlib import contextmanager
 from datetime import datetime
 from functools import partial
 import ConfigParser
+import json
 import tempfile
 
 import pytest
@@ -563,6 +564,229 @@ class TestJobCredentials(TestJobExecution):
         self.task.run_pexpect = mock.Mock(side_effect=run_pexpect_side_effect)
         self.task.run(self.pk)
 
+    def test_custom_environment_injectors_with_jinja_syntax_error(self):
+        some_cloud = CredentialType(
+            kind='cloud',
+            name='SomeCloud',
+            managed_by_tower=False,
+            inputs={
+                'fields': [{
+                    'id': 'api_token',
+                    'label': 'API Token',
+                    'type': 'string'
+                }]
+            },
+            injectors={
+                'env': {
+                    'MY_CLOUD_API_TOKEN': '{{api_token.foo()}}'
+                }
+            }
+        )
+        self.instance.cloud_credential = Credential(
+            credential_type=some_cloud,
+            inputs = {'api_token': 'ABC123'}
+        )
+        with pytest.raises(Exception):
+            self.task.run(self.pk)
+
+    def test_custom_environment_injectors(self):
+        some_cloud = CredentialType(
+            kind='cloud',
+            name='SomeCloud',
+            managed_by_tower=False,
+            inputs={
+                'fields': [{
+                    'id': 'api_token',
+                    'label': 'API Token',
+                    'type': 'string'
+                }]
+            },
+            injectors={
+                'env': {
+                    'MY_CLOUD_API_TOKEN': '{{api_token}}'
+                }
+            }
+        )
+        self.instance.cloud_credential = Credential(
+            credential_type=some_cloud,
+            inputs = {'api_token': 'ABC123'}
+        )
+        self.task.run(self.pk)
+
+        assert self.task.run_pexpect.call_count == 1
+        call_args, _ = self.task.run_pexpect.call_args_list[0]
+        job, args, cwd, env, passwords, stdout = call_args
+
+        assert env['MY_CLOUD_API_TOKEN'] == 'ABC123'
+
+    def test_custom_environment_injectors_with_reserved_env_var(self):
+        some_cloud = CredentialType(
+            kind='cloud',
+            name='SomeCloud',
+            managed_by_tower=False,
+            inputs={
+                'fields': [{
+                    'id': 'api_token',
+                    'label': 'API Token',
+                    'type': 'string'
+                }]
+            },
+            injectors={
+                'env': {
+                    'JOB_ID': 'reserved'
+                }
+            }
+        )
+        self.instance.cloud_credential = Credential(
+            credential_type=some_cloud,
+            inputs = {'api_token': 'ABC123'}
+        )
+        self.task.run(self.pk)
+
+        assert self.task.run_pexpect.call_count == 1
+        call_args, _ = self.task.run_pexpect.call_args_list[0]
+        job, args, cwd, env, passwords, stdout = call_args
+
+        assert env['JOB_ID'] == str(self.instance.pk)
+
+    def test_custom_environment_injectors_with_secret_field(self):
+        some_cloud = CredentialType(
+            kind='cloud',
+            name='SomeCloud',
+            managed_by_tower=False,
+            inputs={
+                'fields': [{
+                    'id': 'password',
+                    'label': 'Password',
+                    'type': 'string',
+                    'secret': True
+                }]
+            },
+            injectors={
+                'env': {
+                    'MY_CLOUD_PRIVATE_VAR': '{{password}}'
+                }
+            }
+        )
+        self.instance.cloud_credential = Credential(
+            credential_type=some_cloud,
+            inputs = {'password': 'SUPER-SECRET-123'}
+        )
+        self.instance.cloud_credential.inputs['password'] = encrypt_field(
+            self.instance.cloud_credential, 'password'
+        )
+        self.task.run(self.pk)
+
+        assert self.task.run_pexpect.call_count == 1
+        call_args, _ = self.task.run_pexpect.call_args_list[0]
+        job, args, cwd, env, passwords, stdout = call_args
+
+        assert env['MY_CLOUD_PRIVATE_VAR'] == 'SUPER-SECRET-123'
+        assert 'SUPER-SECRET-123' not in json.dumps(self.task.update_model.call_args_list)
+
+    def test_custom_environment_injectors_with_extra_vars(self):
+        some_cloud = CredentialType(
+            kind='cloud',
+            name='SomeCloud',
+            managed_by_tower=False,
+            inputs={
+                'fields': [{
+                    'id': 'api_token',
+                    'label': 'API Token',
+                    'type': 'string'
+                }]
+            },
+            injectors={
+                'extra_vars': {
+                    'api_token': '{{api_token}}'
+                }
+            }
+        )
+        self.instance.cloud_credential = Credential(
+            credential_type=some_cloud,
+            inputs = {'api_token': 'ABC123'}
+        )
+        self.task.run(self.pk)
+
+        assert self.task.run_pexpect.call_count == 1
+        call_args, _ = self.task.run_pexpect.call_args_list[0]
+        job, args, cwd, env, passwords, stdout = call_args
+
+        assert '-e {"api_token": "ABC123"}' in ' '.join(args)
+
+    def test_custom_environment_injectors_with_secret_extra_vars(self):
+        """
+        extra_vars that contain secret field values should be censored in the DB
+        """
+        some_cloud = CredentialType(
+            kind='cloud',
+            name='SomeCloud',
+            managed_by_tower=False,
+            inputs={
+                'fields': [{
+                    'id': 'password',
+                    'label': 'Password',
+                    'type': 'string',
+                    'secret': True
+                }]
+            },
+            injectors={
+                'extra_vars': {
+                    'password': '{{password}}'
+                }
+            }
+        )
+        self.instance.cloud_credential = Credential(
+            credential_type=some_cloud,
+            inputs = {'password': 'SUPER-SECRET-123'}
+        )
+        self.instance.cloud_credential.inputs['password'] = encrypt_field(
+            self.instance.cloud_credential, 'password'
+        )
+        self.task.run(self.pk)
+
+        assert self.task.run_pexpect.call_count == 1
+        call_args, _ = self.task.run_pexpect.call_args_list[0]
+        job, args, cwd, env, passwords, stdout = call_args
+
+        assert '-e {"password": "SUPER-SECRET-123"}' in ' '.join(args)
+        assert 'SUPER-SECRET-123' not in json.dumps(self.task.update_model.call_args_list)
+
+    def test_custom_environment_injectors_with_file(self):
+        some_cloud = CredentialType(
+            kind='cloud',
+            name='SomeCloud',
+            managed_by_tower=False,
+            inputs={
+                'fields': [{
+                    'id': 'api_token',
+                    'label': 'API Token',
+                    'type': 'string'
+                }]
+            },
+            injectors={
+                'file': {
+                    'template': '[mycloud]\n{{api_token}}'
+                },
+                'env': {
+                    'MY_CLOUD_INI_FILE': '{{tower.filename}}'
+                }
+            }
+        )
+        self.instance.cloud_credential = Credential(
+            credential_type=some_cloud,
+            inputs = {'api_token': 'ABC123'}
+        )
+        self.task.run(self.pk)
+
+        def run_pexpect_side_effect(*args, **kwargs):
+            job, args, cwd, env, passwords, stdout = args
+            assert open(env['MY_CLOUD_INI_FILE'], 'rb').read() == '[mycloud]\nABC123'
+            return ['successful', 0]
+
+        self.task.run_pexpect = mock.Mock(side_effect=run_pexpect_side_effect)
+        self.task.run(self.pk)
+
 
 class TestProjectUpdateCredentials(TestJobExecution):