fixup return values for bulk launch and host create in awxkit

Enabled the params bulk job

make black

make black again

Fixed inventory and organization input params for bulk modules

add collection integration tests

Fix cli return errors

fix test completeness

awx/api/serializers.py

@@ -1955,7 +1955,7 @@ class BulkHostSerializer(HostSerializer):
     instance_id = serializers.CharField(required=False, max_length=1024)
     description = serializers.CharField(required=False)
     enabled = serializers.BooleanField(default=True, required=False)
-    variables = serializers.CharField(allow_blank=True, required=False)
+    variables = serializers.CharField(write_only=True, required=False)
 
     class Meta:
         fields = (
@@ -4565,6 +4565,7 @@ class BulkJobNodeSerializer(serializers.Serializer):
     survey_passwords = serializers.CharField(required=False, write_only=True, allow_blank=False)
     job_slice_count = serializers.IntegerField(required=False, min_value=1)
     timeout = serializers.IntegerField(required=False, min_value=1)
+    extra_data = serializers.JSONField(write_only=True, required=False)
 
     class Meta:
         fields = (
@@ -4689,11 +4690,16 @@ class BulkJobLaunchSerializer(BaseSerializer):
         job_node_data = validated_data.pop('jobs')
         # FIXME: Need to set organization on the WorkflowJob in order for users to be able to see it --
         # normally their permission is sourced from the underlying WorkflowJobTemplate
-        # maybe we need to add Organization to WorkflowJobd
-        wfj_limit = validated_data.pop('limit', None)
+        # maybe we need to add Organization to WorkflowJob
+        wfj_deferred_attr_names = ('skip_tags', 'limit', 'job_tags')
+        wfj_deferred_vals = {}
+        for item in wfj_deferred_attr_names:
+            wfj_deferred_vals[item] = validated_data.pop(item, None)
+
         wfj = WorkflowJob.objects.create(**validated_data, is_bulk_job=True)
-        if wfj_limit:
-            wfj.limit = wfj_limit
+        for key, val in wfj_deferred_vals.items():
+            if val:
+                setattr(wfj, key, val)
         nodes = []
         node_m2m_objects = {}
         node_m2m_object_types_to_through_model = {
@@ -4717,7 +4723,6 @@ class BulkJobLaunchSerializer(BaseSerializer):
         )
         node_deferred_attrs = {}
         for node_attrs in job_node_data:
-
             # we need to add any m2m objects after creation via the through model
             node_m2m_objects[node_attrs['identifier']] = {}
             node_deferred_attrs[node_attrs['identifier']] = {}

awx/main/migrations/0175_workflowjob_is_bulk_job.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('main', '0174_ensure_org_ee_admin_roles'),
     ]

awx_collection/meta/runtime.yml

@@ -6,6 +6,8 @@ action_groups:
     - ad_hoc_command_cancel
     - ad_hoc_command_wait
     - application
+    - bulk_job_launch
+    - bulk_host_create
     - controller_meta
     - credential_input_source
     - credential

awx_collection/plugins/modules/bulk_host_create.py

@@ -16,22 +16,43 @@ author: "Seth Foster (@fosterseth)"
 short_description: Bulk host create in Automation Platform Controller
 description:
     - Single-request bulk host creation in Automation Platform Controller.
-    - Designed to efficiently add many hosts to an inventory.
+    - Provides a way to add many hosts at once to an inventory in Controller.
 options:
     hosts:
       description:
         - List of hosts to add to inventory.
       required: True
-      type: str
+      type: list
+      elements: dict
+      suboptions:
+        name:
+          description:
+            - The name to use for the host.
+          type: str
+          require: True
+        description:
+          - The description to use for the host.
+        type: str
+        enabled:
+          description:
+            - If the host should be enabled.
+          type: bool
+        variables:
+          description:
+            - Variables to use for the host.
+          type: dict
+        instance_id:
+          description:
+            - instance_id to use for the host.
+          type: str
     inventory:
       description:
-        - Inventory the hosts should be made a member of.
+        - Inventory ID the hosts should be made a member of.
       required: True
-      type: str
+      type: int
 extends_documentation_fragment: awx.awx.auth
 '''
 
-
 EXAMPLES = '''
 - name: Bulk host create
   bulk_host_create:
@@ -44,11 +65,12 @@ EXAMPLES = '''
 from ..module_utils.controller_api import ControllerAPIModule
 import json
 
+
 def main():
     # Any additional arguments that are not fields of the item can be added here
     argument_spec = dict(
         hosts=dict(required=True, type='list'),
-        inventory=dict(),
+        inventory=dict(required=True, type='int'),
     )
 
     # Create a module for ourselves
@@ -58,6 +80,9 @@ def main():
     inventory = module.params.get('inventory')
     hosts = module.params.get('hosts')
 
+    for h in hosts:
+      if 'variables' in h:
+        h['variables'] = json.dumps(h['variables'])
     # Launch the jobs
     result = module.post_endpoint("bulk/host_create", data={"inventory": inventory, "hosts": hosts})
 
@@ -70,4 +95,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    main()

awx_collection/plugins/modules/bulk_job_launch.py

@@ -16,44 +16,134 @@ author: "Seth Foster (@fosterseth)"
 short_description: Bulk job launch in Automation Platform Controller
 description:
     - Single-request bulk job launch in Automation Platform Controller.
-    - The result is flat workflow, each job specified in the parameter jobs results in a workflow job node.
+    - Creates a workflow where each node corresponds to an item specified in the jobs option.
     - Any options specified at the top level will inherited by the launched jobs (if prompt on launch is enabled for those fields).
-    - Designed to efficiently start many jobs at once.
+    - Provides a way to submit many jobs at once to Controller.
 options:
     jobs:
       description:
         - List of jobs to create.
-        - Any promptable field on unified_job_template can be provided as a field on the list item (e.g. limit).
       required: True
       type: list
+      elements: dict
+      suboptions:
+        unified_job_template:
+          description:
+            - Job template ID to use when launching.
+          type: int
+          required: True
+        inventory:
+          description:
+            - Inventory ID applied as a prompt, if job template prompts for inventory
+          type: int
+        execution_environment:
+          description:
+            - Execution environment ID applied as a prompt, if job template prompts for execution environments
+          type: int
+        instance_groups:
+          description:
+            - Instance group IDs applied as a prompt, if job template prompts for instance groups
+          type: list
+          elements: int
+        credentials:
+          description:
+            - Credential IDs applied as a prompt, if job template prompts for credentials
+          type: list
+          elements: int
+        labels:
+          description:
+            - Label IDs to use for the job, if job template prompts for labels
+          type: list
+          elements: int
+        extra_data:
+          description:
+            - Extra variables to apply at launch time, if job template prompts for extra variables
+          type: dict
+          default: {}
+        diff_mode:
+          description:
+            - Show the changes made by Ansible tasks where supported
+          type: bool
+        verbosity:
+          description:
+            - Verbosity level for this ad hoc command run
+          type: int
+          choices: [ 0, 1, 2, 3, 4, 5 ]
+        scm_branch:
+          description:
+            - SCM branch applied as a prompt, if job template prompts for SCM branch
+            - This is only applicable if the project allows for branch override
+          type: str
+        job_type:
+          description:
+            - Job type applied as a prompt, if job template prompts for job type
+          type: str
+          choices:
+            - 'run'
+            - 'check'
+        job_tags:
+          description:
+            - Job tags applied as a prompt, if job template prompts for job tags
+          type: str
+        skip_tags:
+          description:
+            - Tags to skip, applied as a prompt, if job template prompts for job tags
+          type: str
+        limit:
+          description:
+            - Limit to act on, applied as a prompt, if job template prompts for limit
+          type: str
+        forks:
+          description:
+            - The number of parallel or simultaneous processes to use while executing the playbook, if job template prompts for forks
+          type: int
+        job_slice_count:
+          description:
+            - The number of jobs to slice into at runtime, if job template prompts for job slices.
+            - Will cause the Job Template to launch a workflow if value is greater than 1.
+          type: int
+          default: '1'
+        identifier:
+          description:
+            - Identifier for the resulting workflow node that represents this job
+          type: str
+        timeout:
+          description:
+            - Maximum time in seconds to wait for a job to finish (server-side), if job template prompts for timeout.
+          type: int
     name:
       description:
         - The name of the bulk job that is created
       required: False
       type: str
+    description:
+      description:
+        - Optional description of this bulk job.
+      type: str
     organization:
       description:
         - If not provided, will use the organization the user is in.
         - Required if the user belongs to more than one organization.
         - Affects who can see the resulting bulk job.
-      type: str
+      type: int
     inventory:
       description:
-        - Inventory to use for the jobs ran within the bulk job, only used if prompt for inventory is set.
-      type: str
-    limit:
-      description:
-        - Limit to use for the I(job_template).
-      type: str
+        - Inventory ID to use for the jobs ran within the bulk job, only used if prompt for inventory is set.
+      type: int
     scm_branch:
       description:
         - A specific branch of the SCM project to run the template on.
-        - This is only applicable if your project allows for branch override.
+        - This is only applicable if the project allows for branch override.
       type: str
     extra_vars:
       description:
         - Any extra vars required to launch the job.
+        - Extends the extra_data field at the individual job level.
       type: dict
+    limit:
+      description:
+        - Limit to use for the bulk job.
+      type: str
     job_tags:
       description:
         - A comma-separated list of playbook tags to specify what parts of the playbooks should be executed.
@@ -64,7 +154,7 @@ options:
       type: str
     wait:
       description:
-        - Wait for the workflow to complete.
+        - Wait for the bulk job to complete.
       default: True
       type: bool
     interval:
@@ -73,32 +163,33 @@ options:
       required: False
       default: 2
       type: float
-    timeout:
-      description:
-        - If waiting for the workflow to complete this will abort after this
-          amount of seconds
-      type: int
 extends_documentation_fragment: awx.awx.auth
 '''
 
 RETURN = '''
 job_info:
-    description: dictionary containing information about the workflow executed
-    returned: If workflow launched
+    description: dictionary containing information about the bulk job executed
+    returned: If bulk job launched
     type: dict
 '''
 
-
 EXAMPLES = '''
 - name: Launch bulk jobs
   bulk_job_launch:
     name: My Bulk Job Launch
     jobs:
       - unified_job_template: 7
+        skip_tags: foo
       - unified_job_template: 10
         limit: foo
+        extra_data:
+          food: carrot
+          color: orange
     limit: bar
-    inventory: 1 # only affects job templates with prompt on launch enabled for inventory
+    extra_vars: # these override / extend extra_data at the job level
+      food: grape
+      animal: owl
+    inventory: 1
 
 - name: Launch bulk jobs with lookup plugin
   bulk_job_launch:
@@ -111,13 +202,14 @@ EXAMPLES = '''
 from ..module_utils.controller_api import ControllerAPIModule
 import json
 
+
 def main():
     # Any additional arguments that are not fields of the item can be added here
     argument_spec = dict(
         jobs=dict(required=True, type='list'),
         name=dict(),
-        organization=dict(),
-        inventory=dict(),
+        organization=dict(type='int'),
+        inventory=dict(type='int'),
         limit=dict(),
         scm_branch=dict(),
         extra_vars=dict(type='dict'),
@@ -131,15 +223,31 @@ def main():
     # Create a module for ourselves
     module = ControllerAPIModule(argument_spec=argument_spec)
 
+    post_data_names = (
+        'jobs',
+        'name',
+        'organization',
+        'inventory',
+        'limit',
+        'scm_branch',
+        'extra_vars',
+        'job_tags',
+        'skip_tags',
+    )
+    post_data = {}
+    for p in post_data_names:
+        val = module.params.get(p)
+        if val:
+            post_data[p] = val
+
     # Extract our parameters
-    name = module.params.get('name')
     wait = module.params.get('wait')
     timeout = module.params.get('timeout')
     interval = module.params.get('interval')
-    jobs = module.params.get('jobs')
+    name = module.params.get('name')
 
     # Launch the jobs
-    result = module.post_endpoint("bulk/job_launch", data={"jobs": jobs})
+    result = module.post_endpoint("bulk/job_launch", data=post_data)
 
     if result['status_code'] != 201:
         module.fail_json(msg="Failed to launch bulk jobs, see response for details", response=result)
@@ -148,7 +256,7 @@ def main():
     module.json_output['id'] = result['json']['id']
     module.json_output['status'] = result['json']['status']
     # This is for backwards compatability
-    module.json_output['job_info'] = {'id': result['json']['id']}
+    module.json_output['job_info'] = result['json']
 
     if not wait:
         module.exit_json(**module.json_output)
@@ -160,4 +268,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    main()

awx_collection/test/awx/test_bulk.py

@@ -0,0 +1,42 @@
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+import pytest
+
+from awx.main.models import WorkflowJob
+
+@pytest.mark.django_db
+def test_bulk_job_launch(run_module, admin_user, job_template):
+    jobs = [dict(unified_job_template=job_template.id)]
+    result = run_module(
+        'bulk_job_launch',
+        {
+            'name': "foo-bulk-job",
+            'jobs': jobs,
+            'extra_vars': {'animal': 'owl'},
+            'limit': 'foo',
+            'wait': False,
+        },
+        admin_user,
+    )
+
+    bulk_job = WorkflowJob.objects.get(name="foo-bulk-job")
+    assert bulk_job.extra_vars == '{"animal": "owl"}'
+    assert bulk_job.limit == "foo"
+
+
+@pytest.mark.django_db
+def test_bulk_host_create(run_module, admin_user, inventory):
+    hosts = [dict(name="127.0.0.1"), dict(name="foo.dns.org")]
+    result = run_module(
+        'bulk_host_create',
+        {
+            'inventory': inventory.id,
+            'hosts': hosts,
+        },
+        admin_user,
+    )
+    resp_hosts = inventory.hosts.all().values_list('name', flat=True)
+    for h in hosts:
+        assert h['name'] in resp_hosts

awx_collection/test/awx/test_completeness.py

@@ -44,6 +44,12 @@ no_endpoint_for_module = [
     'subscriptions',  # Subscription deals with config/subscriptions
 ]
 
+# Add modules with endpoints that are not at /api/v2
+extra_endpoints = {
+    'bulk_job_launch': '/api/v2/bulk/job_launch/',
+    'bulk_host_create': '/api/v2/bulk/host_create/',
+}
+
 # Global module parameters we can ignore
 ignore_parameters = ['state', 'new_name', 'update_secrets', 'copy_from']
 
@@ -73,6 +79,8 @@ no_api_parameter_ok = {
     'user': ['new_username', 'organization'],
     # workflow_approval parameters that do not apply when approving an approval node.
     'workflow_approval': ['action', 'interval', 'timeout', 'workflow_job_id'],
+    # bulk
+    'bulk_job_launch': ['interval', 'wait'],
 }
 
 # When this tool was created we were not feature complete. Adding something in here indicates a module
@@ -228,6 +236,10 @@ def test_completeness(collection_import, request, admin_user, job_template, exec
         user=admin_user,
         expect=None,
     )
+
+    for key, val in extra_endpoints.items():
+        endpoint_response.data[key] = val
+
     for endpoint in endpoint_response.data.keys():
         # Module names are singular and endpoints are plural so we need to convert to singular
         singular_endpoint = '{0}'.format(endpoint)

awx_collection/tests/integration/targets/bulk_host_create/main.yml

@@ -0,0 +1,51 @@
+---
+- name: Generate a random string for test
+  set_fact:
+    test_id: "{{ lookup('password', '/dev/null chars=ascii_letters length=16') }}"
+  when: test_id is not defined
+
+- name: Generate a unique name
+  set_fact:
+    bulk_host_name: "AWX-Collection-tests-bulk_host_create-{{ test_id }}"
+  
+- name: Get our collection package
+  controller_meta:
+  register: controller_meta
+
+- name: Generate the name of our plugin
+  set_fact:
+    plugin_name: "{{ controller_meta.prefix }}.controller_api"
+
+
+- name: Create an inventory
+  inventory:
+    name: "{{ bulk_host_name }}"
+    organization: Default
+    state: present
+  register: inventory_result
+  
+
+- name: Bulk Host Create
+  bulk_host_create:
+    hosts:
+      - name: "123.456.789.123"
+        description: "myhost1"
+        variables:
+          food: carrot
+          color: orange
+      - name: example.dns.gg
+        description: "myhost2"
+        enabled: false
+    inventory: "{{ inventory_result.id }}"
+  register: result
+
+- assert:
+    that:
+      - result is not failed
+
+# cleanup
+- name: Delete inventory
+  inventory:
+    name: "{{ bulk_host_name }}"
+    organization: Default
+    state: absent

awx_collection/tests/integration/targets/bulk_job_launch/main.yml

@@ -0,0 +1,69 @@
+---
+- name: Generate a random string for test
+  set_fact:
+    test_id: "{{ lookup('password', '/dev/null chars=ascii_letters length=16') }}"
+  when: test_id is not defined
+
+- name: Generate a unique name
+  set_fact:
+    bulk_job_name: "AWX-Collection-tests-bulk_job_launch-{{ test_id }}"
+  
+- name: Get our collection package
+  controller_meta:
+  register: controller_meta
+
+- name: Generate the name of our plugin
+  set_fact:
+    plugin_name: "{{ controller_meta.prefix }}.controller_api"
+
+- name: Get Inventory
+  set_fact:
+    inventory_id: "{{ lookup(plugin_name, 'inventories', query_params={'name': 'Demo Inventory'}, return_ids=True ) }}"
+
+- name: Create a Job Template
+  job_template:
+    name: "{{ bulk_job_name }}"
+    copy_from: "Demo Job Template"
+    ask_variables_on_launch: true
+    ask_inventory_on_launch: true
+    ask_skip_tags_on_launch: true
+    allow_simultaneous: true
+    state: present
+  register: jt_result
+
+- name: Create Bulk Job
+  bulk_job_launch:
+    name: "{{ bulk_job_name }}"
+    jobs:
+      - unified_job_template: "{{ jt_result.id }}"
+        inventory: "{{ inventory_id }}"
+        skip_tags: "skipfoo,skipbar"
+        extra_data:
+          animal: fish
+          color: orange
+      - unified_job_template: "{{ jt_result.id }}"
+    extra_vars:
+      animal: bear
+      food: carrot
+    skip_tags: "skipbaz"
+    job_tags: "Hello World"
+    limit: "localhost"
+    wait: False
+    inventory: "{{ inventory_id }}"
+  register: result
+
+- assert:
+    that:
+      - result is not failed
+      - "'id' in result"
+      - result['job_info']['skip_tags'] == "skipbaz"
+      - result['job_info']['limit'] == "localhost"
+      - result['job_info']['job_tags'] == "Hello World"
+      - result['job_info']['inventory'] == {{ inventory_id }}
+      - "result['job_info']['extra_vars'] == '{\"animal\": \"bear\", \"food\": \"carrot\"}'"
+
+# cleanup
+- name: Delete Job Template
+  job_template:
+    name: "{{ bulk_job_name }}"
+    state: absent

awxkit/awxkit/api/pages/bulk.py

@@ -10,3 +10,12 @@ class Bulk(base.Base):
 
 
 page.register_page([resources.bulk, (resources.bulk, 'get')], Bulk)
+
+
+class BulkJobLaunch(base.Base):
+    def post(self, payload={}):
+        result = self.connection.post(self.endpoint, payload)
+        return self.walk(result.json()['url'])
+
+
+page.register_page(resources.bulk_job_launch, BulkJobLaunch)

awxkit/awxkit/api/resources.py

@@ -14,6 +14,7 @@ class Resources(object):
     _auth = 'auth/'
     _authtoken = 'authtoken/'
     _bulk = 'bulk/'
+    _bulk_job_launch = 'bulk/job_launch/'
     _config = 'config/'
     _config_attach = 'config/attach/'
     _credential = r'credentials/\d+/'