From 9e037f1a0224a13183e6cd4553cf78bd7b9e341c Mon Sep 17 00:00:00 2001 From: Elijah DeLee Date: Fri, 3 Feb 2023 10:08:24 -0500 Subject: [PATCH] fixup return values for bulk launch and host create in awxkit Enabled the params bulk job make black make black again Fixed inventory and organization input params for bulk modules add collection integration tests Fix cli return errors fix test completeness --- awx/api/serializers.py | 17 +- .../0175_workflowjob_is_bulk_job.py | 1 - awx_collection/meta/runtime.yml | 2 + .../plugins/modules/bulk_host_create.py | 39 ++++- .../plugins/modules/bulk_job_launch.py | 164 +++++++++++++++--- awx_collection/test/awx/test_bulk.py | 42 +++++ awx_collection/test/awx/test_completeness.py | 12 ++ .../targets/bulk_host_create/main.yml | 51 ++++++ .../targets/bulk_job_launch/main.yml | 69 ++++++++ awxkit/awxkit/api/pages/bulk.py | 9 + awxkit/awxkit/api/resources.py | 1 + 11 files changed, 365 insertions(+), 42 deletions(-) create mode 100644 awx_collection/test/awx/test_bulk.py create mode 100644 awx_collection/tests/integration/targets/bulk_host_create/main.yml create mode 100644 awx_collection/tests/integration/targets/bulk_job_launch/main.yml diff --git a/awx/api/serializers.py b/awx/api/serializers.py index a19c9bd406..db4888b31f 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1955,7 +1955,7 @@ class BulkHostSerializer(HostSerializer): instance_id = serializers.CharField(required=False, max_length=1024) description = serializers.CharField(required=False) enabled = serializers.BooleanField(default=True, required=False) - variables = serializers.CharField(allow_blank=True, required=False) + variables = serializers.CharField(write_only=True, required=False) class Meta: fields = ( @@ -4565,6 +4565,7 @@ class BulkJobNodeSerializer(serializers.Serializer): survey_passwords = serializers.CharField(required=False, write_only=True, allow_blank=False) job_slice_count = serializers.IntegerField(required=False, min_value=1) timeout = serializers.IntegerField(required=False, min_value=1) + extra_data = serializers.JSONField(write_only=True, required=False) class Meta: fields = ( @@ -4689,11 +4690,16 @@ class BulkJobLaunchSerializer(BaseSerializer): job_node_data = validated_data.pop('jobs') # FIXME: Need to set organization on the WorkflowJob in order for users to be able to see it -- # normally their permission is sourced from the underlying WorkflowJobTemplate - # maybe we need to add Organization to WorkflowJobd - wfj_limit = validated_data.pop('limit', None) + # maybe we need to add Organization to WorkflowJob + wfj_deferred_attr_names = ('skip_tags', 'limit', 'job_tags') + wfj_deferred_vals = {} + for item in wfj_deferred_attr_names: + wfj_deferred_vals[item] = validated_data.pop(item, None) + wfj = WorkflowJob.objects.create(**validated_data, is_bulk_job=True) - if wfj_limit: - wfj.limit = wfj_limit + for key, val in wfj_deferred_vals.items(): + if val: + setattr(wfj, key, val) nodes = [] node_m2m_objects = {} node_m2m_object_types_to_through_model = { @@ -4717,7 +4723,6 @@ class BulkJobLaunchSerializer(BaseSerializer): ) node_deferred_attrs = {} for node_attrs in job_node_data: - # we need to add any m2m objects after creation via the through model node_m2m_objects[node_attrs['identifier']] = {} node_deferred_attrs[node_attrs['identifier']] = {} diff --git a/awx/main/migrations/0175_workflowjob_is_bulk_job.py b/awx/main/migrations/0175_workflowjob_is_bulk_job.py index 8e6c835e8a..c1e71afc73 100644 --- a/awx/main/migrations/0175_workflowjob_is_bulk_job.py +++ b/awx/main/migrations/0175_workflowjob_is_bulk_job.py @@ -4,7 +4,6 @@ from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ('main', '0174_ensure_org_ee_admin_roles'), ] diff --git a/awx_collection/meta/runtime.yml b/awx_collection/meta/runtime.yml index e59becd4ea..3e50a52e69 100644 --- a/awx_collection/meta/runtime.yml +++ b/awx_collection/meta/runtime.yml @@ -6,6 +6,8 @@ action_groups: - ad_hoc_command_cancel - ad_hoc_command_wait - application + - bulk_job_launch + - bulk_host_create - controller_meta - credential_input_source - credential diff --git a/awx_collection/plugins/modules/bulk_host_create.py b/awx_collection/plugins/modules/bulk_host_create.py index f87045b487..3e6ea91cc5 100644 --- a/awx_collection/plugins/modules/bulk_host_create.py +++ b/awx_collection/plugins/modules/bulk_host_create.py @@ -16,22 +16,43 @@ author: "Seth Foster (@fosterseth)" short_description: Bulk host create in Automation Platform Controller description: - Single-request bulk host creation in Automation Platform Controller. - - Designed to efficiently add many hosts to an inventory. + - Provides a way to add many hosts at once to an inventory in Controller. options: hosts: description: - List of hosts to add to inventory. required: True - type: str + type: list + elements: dict + suboptions: + name: + description: + - The name to use for the host. + type: str + require: True + description: + - The description to use for the host. + type: str + enabled: + description: + - If the host should be enabled. + type: bool + variables: + description: + - Variables to use for the host. + type: dict + instance_id: + description: + - instance_id to use for the host. + type: str inventory: description: - - Inventory the hosts should be made a member of. + - Inventory ID the hosts should be made a member of. required: True - type: str + type: int extends_documentation_fragment: awx.awx.auth ''' - EXAMPLES = ''' - name: Bulk host create bulk_host_create: @@ -44,11 +65,12 @@ EXAMPLES = ''' from ..module_utils.controller_api import ControllerAPIModule import json + def main(): # Any additional arguments that are not fields of the item can be added here argument_spec = dict( hosts=dict(required=True, type='list'), - inventory=dict(), + inventory=dict(required=True, type='int'), ) # Create a module for ourselves @@ -58,6 +80,9 @@ def main(): inventory = module.params.get('inventory') hosts = module.params.get('hosts') + for h in hosts: + if 'variables' in h: + h['variables'] = json.dumps(h['variables']) # Launch the jobs result = module.post_endpoint("bulk/host_create", data={"inventory": inventory, "hosts": hosts}) @@ -70,4 +95,4 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/awx_collection/plugins/modules/bulk_job_launch.py b/awx_collection/plugins/modules/bulk_job_launch.py index e6e8c7b103..7a6e800823 100644 --- a/awx_collection/plugins/modules/bulk_job_launch.py +++ b/awx_collection/plugins/modules/bulk_job_launch.py @@ -16,44 +16,134 @@ author: "Seth Foster (@fosterseth)" short_description: Bulk job launch in Automation Platform Controller description: - Single-request bulk job launch in Automation Platform Controller. - - The result is flat workflow, each job specified in the parameter jobs results in a workflow job node. + - Creates a workflow where each node corresponds to an item specified in the jobs option. - Any options specified at the top level will inherited by the launched jobs (if prompt on launch is enabled for those fields). - - Designed to efficiently start many jobs at once. + - Provides a way to submit many jobs at once to Controller. options: jobs: description: - List of jobs to create. - - Any promptable field on unified_job_template can be provided as a field on the list item (e.g. limit). required: True type: list + elements: dict + suboptions: + unified_job_template: + description: + - Job template ID to use when launching. + type: int + required: True + inventory: + description: + - Inventory ID applied as a prompt, if job template prompts for inventory + type: int + execution_environment: + description: + - Execution environment ID applied as a prompt, if job template prompts for execution environments + type: int + instance_groups: + description: + - Instance group IDs applied as a prompt, if job template prompts for instance groups + type: list + elements: int + credentials: + description: + - Credential IDs applied as a prompt, if job template prompts for credentials + type: list + elements: int + labels: + description: + - Label IDs to use for the job, if job template prompts for labels + type: list + elements: int + extra_data: + description: + - Extra variables to apply at launch time, if job template prompts for extra variables + type: dict + default: {} + diff_mode: + description: + - Show the changes made by Ansible tasks where supported + type: bool + verbosity: + description: + - Verbosity level for this ad hoc command run + type: int + choices: [ 0, 1, 2, 3, 4, 5 ] + scm_branch: + description: + - SCM branch applied as a prompt, if job template prompts for SCM branch + - This is only applicable if the project allows for branch override + type: str + job_type: + description: + - Job type applied as a prompt, if job template prompts for job type + type: str + choices: + - 'run' + - 'check' + job_tags: + description: + - Job tags applied as a prompt, if job template prompts for job tags + type: str + skip_tags: + description: + - Tags to skip, applied as a prompt, if job template prompts for job tags + type: str + limit: + description: + - Limit to act on, applied as a prompt, if job template prompts for limit + type: str + forks: + description: + - The number of parallel or simultaneous processes to use while executing the playbook, if job template prompts for forks + type: int + job_slice_count: + description: + - The number of jobs to slice into at runtime, if job template prompts for job slices. + - Will cause the Job Template to launch a workflow if value is greater than 1. + type: int + default: '1' + identifier: + description: + - Identifier for the resulting workflow node that represents this job + type: str + timeout: + description: + - Maximum time in seconds to wait for a job to finish (server-side), if job template prompts for timeout. + type: int name: description: - The name of the bulk job that is created required: False type: str + description: + description: + - Optional description of this bulk job. + type: str organization: description: - If not provided, will use the organization the user is in. - Required if the user belongs to more than one organization. - Affects who can see the resulting bulk job. - type: str + type: int inventory: description: - - Inventory to use for the jobs ran within the bulk job, only used if prompt for inventory is set. - type: str - limit: - description: - - Limit to use for the I(job_template). - type: str + - Inventory ID to use for the jobs ran within the bulk job, only used if prompt for inventory is set. + type: int scm_branch: description: - A specific branch of the SCM project to run the template on. - - This is only applicable if your project allows for branch override. + - This is only applicable if the project allows for branch override. type: str extra_vars: description: - Any extra vars required to launch the job. + - Extends the extra_data field at the individual job level. type: dict + limit: + description: + - Limit to use for the bulk job. + type: str job_tags: description: - A comma-separated list of playbook tags to specify what parts of the playbooks should be executed. @@ -64,7 +154,7 @@ options: type: str wait: description: - - Wait for the workflow to complete. + - Wait for the bulk job to complete. default: True type: bool interval: @@ -73,32 +163,33 @@ options: required: False default: 2 type: float - timeout: - description: - - If waiting for the workflow to complete this will abort after this - amount of seconds - type: int extends_documentation_fragment: awx.awx.auth ''' RETURN = ''' job_info: - description: dictionary containing information about the workflow executed - returned: If workflow launched + description: dictionary containing information about the bulk job executed + returned: If bulk job launched type: dict ''' - EXAMPLES = ''' - name: Launch bulk jobs bulk_job_launch: name: My Bulk Job Launch jobs: - unified_job_template: 7 + skip_tags: foo - unified_job_template: 10 limit: foo + extra_data: + food: carrot + color: orange limit: bar - inventory: 1 # only affects job templates with prompt on launch enabled for inventory + extra_vars: # these override / extend extra_data at the job level + food: grape + animal: owl + inventory: 1 - name: Launch bulk jobs with lookup plugin bulk_job_launch: @@ -111,13 +202,14 @@ EXAMPLES = ''' from ..module_utils.controller_api import ControllerAPIModule import json + def main(): # Any additional arguments that are not fields of the item can be added here argument_spec = dict( jobs=dict(required=True, type='list'), name=dict(), - organization=dict(), - inventory=dict(), + organization=dict(type='int'), + inventory=dict(type='int'), limit=dict(), scm_branch=dict(), extra_vars=dict(type='dict'), @@ -131,15 +223,31 @@ def main(): # Create a module for ourselves module = ControllerAPIModule(argument_spec=argument_spec) + post_data_names = ( + 'jobs', + 'name', + 'organization', + 'inventory', + 'limit', + 'scm_branch', + 'extra_vars', + 'job_tags', + 'skip_tags', + ) + post_data = {} + for p in post_data_names: + val = module.params.get(p) + if val: + post_data[p] = val + # Extract our parameters - name = module.params.get('name') wait = module.params.get('wait') timeout = module.params.get('timeout') interval = module.params.get('interval') - jobs = module.params.get('jobs') + name = module.params.get('name') # Launch the jobs - result = module.post_endpoint("bulk/job_launch", data={"jobs": jobs}) + result = module.post_endpoint("bulk/job_launch", data=post_data) if result['status_code'] != 201: module.fail_json(msg="Failed to launch bulk jobs, see response for details", response=result) @@ -148,7 +256,7 @@ def main(): module.json_output['id'] = result['json']['id'] module.json_output['status'] = result['json']['status'] # This is for backwards compatability - module.json_output['job_info'] = {'id': result['json']['id']} + module.json_output['job_info'] = result['json'] if not wait: module.exit_json(**module.json_output) @@ -160,4 +268,4 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/awx_collection/test/awx/test_bulk.py b/awx_collection/test/awx/test_bulk.py new file mode 100644 index 0000000000..fbccf3d1d8 --- /dev/null +++ b/awx_collection/test/awx/test_bulk.py @@ -0,0 +1,42 @@ +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import pytest + +from awx.main.models import WorkflowJob + +@pytest.mark.django_db +def test_bulk_job_launch(run_module, admin_user, job_template): + jobs = [dict(unified_job_template=job_template.id)] + result = run_module( + 'bulk_job_launch', + { + 'name': "foo-bulk-job", + 'jobs': jobs, + 'extra_vars': {'animal': 'owl'}, + 'limit': 'foo', + 'wait': False, + }, + admin_user, + ) + + bulk_job = WorkflowJob.objects.get(name="foo-bulk-job") + assert bulk_job.extra_vars == '{"animal": "owl"}' + assert bulk_job.limit == "foo" + + +@pytest.mark.django_db +def test_bulk_host_create(run_module, admin_user, inventory): + hosts = [dict(name="127.0.0.1"), dict(name="foo.dns.org")] + result = run_module( + 'bulk_host_create', + { + 'inventory': inventory.id, + 'hosts': hosts, + }, + admin_user, + ) + resp_hosts = inventory.hosts.all().values_list('name', flat=True) + for h in hosts: + assert h['name'] in resp_hosts \ No newline at end of file diff --git a/awx_collection/test/awx/test_completeness.py b/awx_collection/test/awx/test_completeness.py index 451c1a61d3..ef3d70727a 100644 --- a/awx_collection/test/awx/test_completeness.py +++ b/awx_collection/test/awx/test_completeness.py @@ -44,6 +44,12 @@ no_endpoint_for_module = [ 'subscriptions', # Subscription deals with config/subscriptions ] +# Add modules with endpoints that are not at /api/v2 +extra_endpoints = { + 'bulk_job_launch': '/api/v2/bulk/job_launch/', + 'bulk_host_create': '/api/v2/bulk/host_create/', +} + # Global module parameters we can ignore ignore_parameters = ['state', 'new_name', 'update_secrets', 'copy_from'] @@ -73,6 +79,8 @@ no_api_parameter_ok = { 'user': ['new_username', 'organization'], # workflow_approval parameters that do not apply when approving an approval node. 'workflow_approval': ['action', 'interval', 'timeout', 'workflow_job_id'], + # bulk + 'bulk_job_launch': ['interval', 'wait'], } # When this tool was created we were not feature complete. Adding something in here indicates a module @@ -228,6 +236,10 @@ def test_completeness(collection_import, request, admin_user, job_template, exec user=admin_user, expect=None, ) + + for key, val in extra_endpoints.items(): + endpoint_response.data[key] = val + for endpoint in endpoint_response.data.keys(): # Module names are singular and endpoints are plural so we need to convert to singular singular_endpoint = '{0}'.format(endpoint) diff --git a/awx_collection/tests/integration/targets/bulk_host_create/main.yml b/awx_collection/tests/integration/targets/bulk_host_create/main.yml new file mode 100644 index 0000000000..c5b0437f37 --- /dev/null +++ b/awx_collection/tests/integration/targets/bulk_host_create/main.yml @@ -0,0 +1,51 @@ +--- +- name: Generate a random string for test + set_fact: + test_id: "{{ lookup('password', '/dev/null chars=ascii_letters length=16') }}" + when: test_id is not defined + +- name: Generate a unique name + set_fact: + bulk_host_name: "AWX-Collection-tests-bulk_host_create-{{ test_id }}" + +- name: Get our collection package + controller_meta: + register: controller_meta + +- name: Generate the name of our plugin + set_fact: + plugin_name: "{{ controller_meta.prefix }}.controller_api" + + +- name: Create an inventory + inventory: + name: "{{ bulk_host_name }}" + organization: Default + state: present + register: inventory_result + + +- name: Bulk Host Create + bulk_host_create: + hosts: + - name: "123.456.789.123" + description: "myhost1" + variables: + food: carrot + color: orange + - name: example.dns.gg + description: "myhost2" + enabled: false + inventory: "{{ inventory_result.id }}" + register: result + +- assert: + that: + - result is not failed + +# cleanup +- name: Delete inventory + inventory: + name: "{{ bulk_host_name }}" + organization: Default + state: absent \ No newline at end of file diff --git a/awx_collection/tests/integration/targets/bulk_job_launch/main.yml b/awx_collection/tests/integration/targets/bulk_job_launch/main.yml new file mode 100644 index 0000000000..6c014e3732 --- /dev/null +++ b/awx_collection/tests/integration/targets/bulk_job_launch/main.yml @@ -0,0 +1,69 @@ +--- +- name: Generate a random string for test + set_fact: + test_id: "{{ lookup('password', '/dev/null chars=ascii_letters length=16') }}" + when: test_id is not defined + +- name: Generate a unique name + set_fact: + bulk_job_name: "AWX-Collection-tests-bulk_job_launch-{{ test_id }}" + +- name: Get our collection package + controller_meta: + register: controller_meta + +- name: Generate the name of our plugin + set_fact: + plugin_name: "{{ controller_meta.prefix }}.controller_api" + +- name: Get Inventory + set_fact: + inventory_id: "{{ lookup(plugin_name, 'inventories', query_params={'name': 'Demo Inventory'}, return_ids=True ) }}" + +- name: Create a Job Template + job_template: + name: "{{ bulk_job_name }}" + copy_from: "Demo Job Template" + ask_variables_on_launch: true + ask_inventory_on_launch: true + ask_skip_tags_on_launch: true + allow_simultaneous: true + state: present + register: jt_result + +- name: Create Bulk Job + bulk_job_launch: + name: "{{ bulk_job_name }}" + jobs: + - unified_job_template: "{{ jt_result.id }}" + inventory: "{{ inventory_id }}" + skip_tags: "skipfoo,skipbar" + extra_data: + animal: fish + color: orange + - unified_job_template: "{{ jt_result.id }}" + extra_vars: + animal: bear + food: carrot + skip_tags: "skipbaz" + job_tags: "Hello World" + limit: "localhost" + wait: False + inventory: "{{ inventory_id }}" + register: result + +- assert: + that: + - result is not failed + - "'id' in result" + - result['job_info']['skip_tags'] == "skipbaz" + - result['job_info']['limit'] == "localhost" + - result['job_info']['job_tags'] == "Hello World" + - result['job_info']['inventory'] == {{ inventory_id }} + - "result['job_info']['extra_vars'] == '{\"animal\": \"bear\", \"food\": \"carrot\"}'" + +# cleanup +- name: Delete Job Template + job_template: + name: "{{ bulk_job_name }}" + state: absent \ No newline at end of file diff --git a/awxkit/awxkit/api/pages/bulk.py b/awxkit/awxkit/api/pages/bulk.py index 197b5c2585..b7342cab45 100644 --- a/awxkit/awxkit/api/pages/bulk.py +++ b/awxkit/awxkit/api/pages/bulk.py @@ -10,3 +10,12 @@ class Bulk(base.Base): page.register_page([resources.bulk, (resources.bulk, 'get')], Bulk) + + +class BulkJobLaunch(base.Base): + def post(self, payload={}): + result = self.connection.post(self.endpoint, payload) + return self.walk(result.json()['url']) + + +page.register_page(resources.bulk_job_launch, BulkJobLaunch) diff --git a/awxkit/awxkit/api/resources.py b/awxkit/awxkit/api/resources.py index 31bf6c5829..448a0bb582 100644 --- a/awxkit/awxkit/api/resources.py +++ b/awxkit/awxkit/api/resources.py @@ -14,6 +14,7 @@ class Resources(object): _auth = 'auth/' _authtoken = 'authtoken/' _bulk = 'bulk/' + _bulk_job_launch = 'bulk/job_launch/' _config = 'config/' _config_attach = 'config/attach/' _credential = r'credentials/\d+/'