Merge pull request #6743 from john-westcott-iv/version_warning

Adding version checking to collection

Reviewed-by: Bianca Henderson <beeankha@gmail.com>
             https://github.com/beeankha

Makefile

@@ -393,7 +393,7 @@ symlink_collection:
 	ln -s $(shell pwd)/awx_collection $(COLLECTION_INSTALL)
 
 build_collection:
-	ansible-playbook -i localhost, awx_collection/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e collection_namespace=$(COLLECTION_NAMESPACE) -e collection_version=$(VERSION)
+	ansible-playbook -i localhost, awx_collection/tools/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e collection_namespace=$(COLLECTION_NAMESPACE) -e collection_version=$(VERSION) -e '{"awx_template_version":false}'
 	ansible-galaxy collection build awx_collection --force --output-path=awx_collection
 
 install_collection: build_collection

awx_collection/plugins/doc_fragments/auth.py

@@ -39,6 +39,7 @@ options:
   tower_config_file:
     description:
     - Path to the Tower or AWX config file.
+    - If provided, the other locations for config files will not be considered.
     type: path
 
 notes:

awx_collection/plugins/module_utils/tower_api.py

@@ -32,6 +32,15 @@ class ItemNotDefined(Exception):
 
 
 class TowerModule(AnsibleModule):
+    # This gets set by the make process so whatever is in here is irrelevant
+    _COLLECTION_VERSION = "devel"
+    _COLLECTION_TYPE = "awx"
+    # This maps the collections type (awx/tower) to the values returned by the API
+    # Those values can be found in awx/api/generics.py line 204
+    collection_to_version = {
+        'awx': 'AWX',
+        'tower': 'Red Hat Ansible Tower',
+    }
     url = None
     honorred_settings = ('host', 'username', 'password', 'verify_ssl', 'oauth_token')
     host = '127.0.0.1'
@@ -45,6 +54,7 @@ class TowerModule(AnsibleModule):
     authenticated = False
     config_name = 'tower_cli.cfg'
     ENCRYPTED_STRING = "$encrypted$"
+    version_checked = False
 
     def __init__(self, argument_spec, **kwargs):
         args = dict(
@@ -104,14 +114,6 @@ class TowerModule(AnsibleModule):
             local_dir = split(local_dir)[0]
             config_files.insert(2, join(local_dir, ".{0}".format(self.config_name)))
 
-        for config_file in config_files:
-            if exists(config_file) and not isdir(config_file):
-                # Only throw a formatting error if the file exists and is not a directory
-                try:
-                    self.load_config(config_file)
-                except ConfigFileException:
-                    self.fail_json('The config file {0} is not properly formatted'.format(config_file))
-
         # If we have a specified  tower config, load it
         if self.params.get('tower_config_file'):
             duplicated_params = []
@@ -129,6 +131,14 @@ class TowerModule(AnsibleModule):
             except ConfigFileException as cfe:
                 # Since we were told specifically to load this we want it to fail if we have an error
                 self.fail_json(msg=cfe)
+        else:
+            for config_file in config_files:
+                if exists(config_file) and not isdir(config_file):
+                    # Only throw a formatting error if the file exists and is not a directory
+                    try:
+                        self.load_config(config_file)
+                    except ConfigFileException:
+                        self.fail_json('The config file {0} is not properly formatted'.format(config_file))
 
     def load_config(self, config_path):
         # Validate the config file is an actual file
@@ -374,6 +384,26 @@ class TowerModule(AnsibleModule):
         finally:
             self.url = self.url._replace(query=None)
 
+        if not self.version_checked:
+            # In PY2 we get back an HTTPResponse object but PY2 is returning an addinfourl
+            # First try to get the headers in PY3 format and then drop down to PY2.
+            try:
+                tower_type = response.getheader('X-API-Product-Name', None)
+                tower_version = response.getheader('X-API-Product-Version', None)
+            except Exception:
+                tower_type = response.info().getheader('X-API-Product-Name', None)
+                tower_version = response.info().getheader('X-API-Product-Version', None)
+
+            if self._COLLECTION_TYPE not in self.collection_to_version or self.collection_to_version[self._COLLECTION_TYPE] != tower_type:
+                self.warn("You are using the {0} version of this collection but connecting to {1}".format(
+                    self._COLLECTION_TYPE, tower_type
+                ))
+            elif self._COLLECTION_VERSION != tower_version:
+                self.warn("You are running collection version {0} but connecting to tower version {1}".format(
+                    self._COLLECTION_VERSION, tower_version
+                ))
+            self.version_checked = True
+
         response_body = ''
         try:
             response_body = response.read()
@@ -434,15 +464,6 @@ class TowerModule(AnsibleModule):
         # If we have neither of these, then we can try un-authenticated access
         self.authenticated = True
 
-    def default_check_mode(self):
-        '''Execute check mode logic for Ansible Tower modules'''
-        if self.check_mode:
-            try:
-                result = self.get_endpoint('ping')
-                self.exit_json(**{'changed': True, 'tower_version': '{0}'.format(result['json']['version'])})
-            except(Exception) as excinfo:
-                self.fail_json(changed=False, msg='Failed check mode: {0}'.format(excinfo))
-
     def delete_if_needed(self, existing_item, on_delete=None):
         # This will exit from the module on its own.
         # If the method successfully deletes an item and on_delete param is defined,

awx_collection/template_galaxy.yml

@@ -1,35 +0,0 @@
----
-- hosts: localhost
-  gather_facts: false
-  connection: local
-  vars:
-    collection_package: awx
-    collection_namespace: awx
-    collection_version: 0.0.1  # not for updating, pass in extra_vars
-
-  tasks:
-    - name: Do file content replacements for non-default namespace or package name
-      block:
-
-        - name: Change module doc_fragments to support desired namespace and package names
-          replace:
-            path: "{{ item }}"
-            regexp: '^extends_documentation_fragment: awx.awx.auth$'
-            replace: 'extends_documentation_fragment: {{ collection_namespace }}.{{ collection_package }}.auth'
-          with_fileglob: "{{ playbook_dir }}/plugins/modules/tower_*.py"
-          loop_control:
-            label: "{{ item | basename }}"
-
-        - name: Change inventory file to support desired namespace and package names
-          replace:
-            path: "{{ playbook_dir }}/plugins/inventory/tower.py"
-            regexp: "^    NAME = 'awx.awx.tower'  # REPLACE$"
-            replace: "    NAME = '{{ collection_namespace }}.{{ collection_package }}.tower'  # REPLACE"
-      when:
-        - (collection_package != 'awx') or (collection_namespace != 'awx')
-
-    - name: Template the galaxy.yml file
-      template:
-        src: "{{ playbook_dir }}/galaxy.yml.j2"
-        dest: "{{ playbook_dir }}/galaxy.yml"
-        force: true

awx_collection/test/awx/conftest.py

@@ -108,6 +108,7 @@ def run_module(request, collection_import):
             sanitize_dict(py_data)
             resp._content = bytes(json.dumps(django_response.data), encoding='utf8')
             resp.status_code = django_response.status_code
+            resp.headers = {'X-API-Product-Name': 'AWX', 'X-API-Product-Version': 'devel'}
 
             if request.config.getoption('verbose') > 0:
                 logger.info(
@@ -120,7 +121,11 @@ def run_module(request, collection_import):
 
         def new_open(self, method, url, **kwargs):
             r = new_request(self, method, url, **kwargs)
-            return mock.MagicMock(read=mock.MagicMock(return_value=r._content), status=r.status_code)
+            m = mock.MagicMock(read=mock.MagicMock(return_value=r._content),
+                               status=r.status_code,
+                               getheader=mock.MagicMock(side_effect=r.headers.get)
+                               )
+            return m
 
         stdout_buffer = io.StringIO()
         # Requies specific PYTHONPATH, see docs
@@ -245,7 +250,7 @@ def silence_deprecation():
         yield this_mock
 
 
-@pytest.fixture
+@pytest.fixture(autouse=True)
 def silence_warning():
     """Warnings use global variable, same as deprecations."""
     with mock.patch('ansible.module_utils.basic.AnsibleModule.warn') as this_mock:

awx_collection/test/awx/test_credential.py

@@ -152,7 +152,7 @@ def test_make_use_of_custom_credential_type(run_module, organization, admin_user
 
 
 @pytest.mark.django_db
-def test_secret_field_write_twice(run_module, organization, admin_user, cred_type, silence_warning):
+def test_secret_field_write_twice(run_module, organization, admin_user, cred_type):
     val1 = '7rEZK38DJl58A7RxA6EC7lLvUHbBQ1'
     result = run_module('tower_credential', dict(
         name='Galaxy Token for Steve',

awx_collection/test/awx/test_job_template.py

@@ -163,8 +163,7 @@ def test_job_template_with_survey_encrypted_default(run_module, admin_user, proj
 
     silence_warning.assert_called_once_with(
         "The field survey_spec of job_template {0} has encrypted data and "
-        "may inaccurately report task is changed.".format(result['id'])
+        "may inaccurately report task is changed.".format(result['id']))
-    )
 
 
 @pytest.mark.django_db

awx_collection/test/awx/test_module_utils.py

@@ -1,14 +1,67 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import json
 import sys
 
+from requests.models import Response
 from unittest import mock
 
-import json
+
+def getheader(self, header_name, default):
+    mock_headers = {'X-API-Product-Name': 'not-junk', 'X-API-Product-Version': '1.2.3'}
+    return mock_headers.get(header_name, default)
 
 
-def test_duplicate_config(collection_import):
+def read(self):
+    return json.dumps({})
+
+
+def status(self):
+    return 200
+
+
+def mock_ping_response(self, method, url, **kwargs):
+    r = Response()
+    r.getheader = getheader.__get__(r)
+    r.read = read.__get__(r)
+    r.status = status.__get__(r)
+    return r
+
+
+def test_version_warning(collection_import, silence_warning):
+    TowerModule = collection_import('plugins.module_utils.tower_api').TowerModule
+    cli_data = {'ANSIBLE_MODULE_ARGS': {}}
+    testargs = ['module_file2.py', json.dumps(cli_data)]
+    with mock.patch.object(sys, 'argv', testargs):
+        with mock.patch('ansible.module_utils.urls.Request.open', new=mock_ping_response):
+            my_module = TowerModule(argument_spec=dict())
+            my_module._COLLECTION_VERSION = "1.0.0"
+            my_module._COLLECTION_TYPE = "not-junk"
+            my_module.collection_to_version['not-junk'] = 'not-junk'
+            my_module.get_endpoint('ping')
+    silence_warning.assert_called_once_with(
+        'You are running collection version 1.0.0 but connecting to tower version 1.2.3'
+    )
+
+
+def test_type_warning(collection_import, silence_warning):
+    TowerModule = collection_import('plugins.module_utils.tower_api').TowerModule
+    cli_data = {'ANSIBLE_MODULE_ARGS': {}}
+    testargs = ['module_file2.py', json.dumps(cli_data)]
+    with mock.patch.object(sys, 'argv', testargs):
+        with mock.patch('ansible.module_utils.urls.Request.open', new=mock_ping_response):
+            my_module = TowerModule(argument_spec={})
+            my_module._COLLECTION_VERSION = "1.2.3"
+            my_module._COLLECTION_TYPE = "junk"
+            my_module.collection_to_version['junk'] = 'junk'
+            my_module.get_endpoint('ping')
+    silence_warning.assert_called_once_with(
+        'You are using the junk version of this collection but connecting to not-junk'
+    )
+
+
+def test_duplicate_config(collection_import, silence_warning):
     # imports done here because of PATH issues unique to this test suite
     TowerModule = collection_import('plugins.module_utils.tower_api').TowerModule
     data = {
@@ -17,19 +70,42 @@ def test_duplicate_config(collection_import):
         'tower_username': 'bob',
         'tower_config_file': 'my_config'
     }
+
+    class DuplicateTestTowerModule(TowerModule):
+        def load_config(self, config_path):
+            assert config_path == 'my_config'
+
+        def _load_params(self):
+            self.params = data
+
     cli_data = {'ANSIBLE_MODULE_ARGS': data}
     testargs = ['module_file.py', json.dumps(cli_data)]
-    with mock.patch('ansible.module_utils.basic.AnsibleModule.warn') as mock_warn:
+    with mock.patch.object(sys, 'argv', testargs):
-        with mock.patch.object(sys, 'argv', testargs):
+        argument_spec = dict(
-            with mock.patch.object(TowerModule, 'load_config') as mock_load:
+            name=dict(required=True),
-                argument_spec = dict(
+            zig=dict(type='str'),
-                    name=dict(required=True),
+        )
-                    zig=dict(type='str'),
+        DuplicateTestTowerModule(argument_spec=argument_spec)
-                )
+    silence_warning.assert_called_once_with(
-                TowerModule(argument_spec=argument_spec)
-            mock_load.mock_calls[-1] == mock.call('my_config')
-    mock_warn.assert_called_once_with(
         'The parameter(s) tower_username were provided at the same time as '
         'tower_config_file. Precedence may be unstable, '
         'we suggest either using config file or params.'
     )
+
+
+def test_no_templated_values(collection_import):
+    """This test corresponds to replacements done by
+    awx_collection/tools/roles/template_galaxy/tasks/main.yml
+    Those replacements should happen at build time, so they should not be
+    checked into source.
+    """
+    TowerModule = collection_import('plugins.module_utils.tower_api').TowerModule
+    assert TowerModule._COLLECTION_VERSION == "devel", (
+        'The collection version is templated when the collection is built '
+        'and the code should retain the placeholder of "devel".'
+    )
+    InventoryModule = collection_import('plugins.inventory.tower').InventoryModule
+    assert InventoryModule.NAME == 'awx.awx.tower', (
+        'The inventory plugin FQCN is templated when the collection is built '
+        'and the code should retain the default of awx.awx.'
+    )

awx_collection/test/awx/test_project.py

@@ -3,23 +3,23 @@ __metaclass__ = type
 
 import pytest
 
-from unittest import mock
-
 from awx.main.models import Project
 
 
 @pytest.mark.django_db
-def test_create_project(run_module, admin_user, organization):
+def test_create_project(run_module, admin_user, organization, silence_warning):
-    with mock.patch('ansible.module_utils.basic.AnsibleModule.warn') as mock_warn:
+    result = run_module('tower_project', dict(
-        result = run_module('tower_project', dict(
+        name='foo',
-            name='foo',
+        organization=organization.name,
-            organization=organization.name,
+        scm_type='git',
-            scm_type='git',
+        scm_url='https://foo.invalid',
-            scm_url='https://foo.invalid',
+        wait=False,
-            wait=False,
+        scm_update_cache_timeout=5
-            scm_update_cache_timeout=5
+    ), admin_user)
-        ), admin_user)
+    silence_warning.assert_called_once_with(
-    mock_warn.assert_called_once_with('scm_update_cache_timeout will be ignored since scm_update_on_launch was not set to true')
+        'scm_update_cache_timeout will be ignored since scm_update_on_launch '
+        'was not set to true')
+
     assert result.pop('changed', None), result
 
     proj = Project.objects.get(name='foo')

awx_collection/test/awx/test_send_receive.py

@@ -17,7 +17,7 @@ from awx.main.models import (
 
 # warns based on password_management param, but not security issue
 @pytest.mark.django_db
-def test_receive_send_jt(run_module, admin_user, mocker, silence_deprecation, silence_warning):
+def test_receive_send_jt(run_module, admin_user, mocker, silence_deprecation):
     org = Organization.objects.create(name='SRtest')
     proj = Project.objects.create(
         name='SRtest',

awx_collection/test/awx/test_user.py

@@ -43,5 +43,4 @@ def test_password_no_op_warning(run_module, admin_user, mock_auth_stuff, silence
 
     silence_warning.assert_called_once_with(
         "The field password of user {0} has encrypted data and "
-        "may inaccurately report task is changed.".format(result['id'])
+        "may inaccurately report task is changed.".format(result['id']))
-    )

awx_collection/tools/generate.yml

@@ -17,68 +17,5 @@
       force_basic_auth: true
       url_username: "{{ lookup('env', 'TOWER_USERNAME') }}"
       url_password: "{{ lookup('env', 'TOWER_PASSWORD') }}"
-
+  roles:
-  tasks:
+    - generate
-    - name: Get date time data
-      setup:
-        gather_subset: min
-
-    - name: Create module directory
-      file:
-        state: directory
-        name: "modules"
-
-    - name: Load api/v2
-      uri:
-        method: GET
-        url: "{{ api_url }}/api/v2/"
-      register: endpoints
-
-    - name: Load endpoint options
-      uri:
-        method: "OPTIONS"
-        url: "{{ api_url }}{{ item.value }}"
-      loop: "{{ endpoints['json'] | dict2items }}"
-      loop_control:
-        label: "{{ item.key }}"
-      register: end_point_options
-      when: "generate_for is not defined or item.key in generate_for"
-
-    - name: Scan POST options for different things
-      set_fact:
-        all_options: "{{ all_options | default({}) | combine(options[0]) }}"
-      loop: "{{ end_point_options.results }}"
-      vars:
-        options: "{{ item | json_query('json.actions.POST.[*]') }}"
-      loop_control:
-        label: "{{ item['item']['key'] }}"
-      when:
-        - item is not skipped
-        - options is defined
-
-    - name: Process endpoint
-      template:
-        src: "templates/tower_module.j2"
-        dest: "{{ playbook_dir | dirname }}/plugins/modules/{{ file_name }}"
-      loop: "{{ end_point_options['results'] }}"
-      loop_control:
-        label: "{{ item['item']['key'] }}"
-      when: "'json' in item and 'actions' in item['json'] and 'POST' in item['json']['actions']"
-      vars:
-        item_type: "{{ item['item']['key'] }}"
-        human_readable: "{{ item_type | replace('_', ' ') }}"
-        singular_item_type: "{{ item['item']['key'] | regex_replace('ies$', 'y') | regex_replace('s$', '') }}"
-        file_name: "tower_{% if item['item']['key'] in ['settings'] %}{{ item['item']['key'] }}{% else %}{{ singular_item_type }}{% endif %}.py"
-        type_map:
-          bool: 'bool'
-          boolean: 'bool'
-          choice: 'str'
-          datetime: 'str'
-          id: 'str'
-          int: 'int'
-          integer: 'int'
-          json: 'dict'
-          list: 'list'
-          object: 'dict'
-          password: 'str'
-          string: 'str'

awx_collection/tools/roles/generate/tasks/main.yml

@@ -0,0 +1,64 @@
+---
+- name: Get date time data
+  setup:
+    gather_subset: min
+
+- name: Create module directory
+  file:
+    state: directory
+    name: "modules"
+
+- name: Load api/v2
+  uri:
+    method: GET
+    url: "{{ api_url }}/api/v2/"
+  register: endpoints
+
+- name: Load endpoint options
+  uri:
+    method: "OPTIONS"
+    url: "{{ api_url }}{{ item.value }}"
+  loop: "{{ endpoints['json'] | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+  register: end_point_options
+  when: "generate_for is not defined or item.key in generate_for"
+
+- name: Scan POST options for different things
+  set_fact:
+    all_options: "{{ all_options | default({}) | combine(options[0]) }}"
+  loop: "{{ end_point_options.results }}"
+  vars:
+    options: "{{ item | json_query('json.actions.POST.[*]') }}"
+  loop_control:
+    label: "{{ item['item']['key'] }}"
+  when:
+    - item is not skipped
+    - options is defined
+
+- name: Process endpoint
+  template:
+    src: "templates/tower_module.j2"
+    dest: "{{ playbook_dir | dirname }}/plugins/modules/{{ file_name }}"
+  loop: "{{ end_point_options['results'] }}"
+  loop_control:
+    label: "{{ item['item']['key'] }}"
+  when: "'json' in item and 'actions' in item['json'] and 'POST' in item['json']['actions']"
+  vars:
+    item_type: "{{ item['item']['key'] }}"
+    human_readable: "{{ item_type | replace('_', ' ') }}"
+    singular_item_type: "{{ item['item']['key'] | regex_replace('ies$', 'y') | regex_replace('s$', '') }}"
+    file_name: "tower_{% if item['item']['key'] in ['settings'] %}{{ item['item']['key'] }}{% else %}{{ singular_item_type }}{% endif %}.py"
+    type_map:
+      bool: 'bool'
+      boolean: 'bool'
+      choice: 'str'
+      datetime: 'str'
+      id: 'str'
+      int: 'int'
+      integer: 'int'
+      json: 'dict'
+      list: 'list'
+      object: 'dict'
+      password: 'str'
+      string: 'str'

awx_collection/tools/roles/template_galaxy/tasks/main.yml

@@ -0,0 +1,40 @@
+---
+- name: Set the collection version in the tower_api.py file
+  replace:
+    path: "{{ collection_path }}/plugins/module_utils/tower_api.py"
+    regexp: '^    _COLLECTION_VERSION = "devel"'
+    replace: '    _COLLECTION_VERSION = "{{ collection_version }}"'
+  when:
+    - "awx_template_version | default(True)"
+
+- name: Set the collection type in the tower_api.py file
+  replace:
+    path: "{{ collection_path }}/plugins/module_utils/tower_api.py"
+    regexp: '^    _COLLECTION_TYPE = "awx"'
+    replace: '    _COLLECTION_TYPE = "{{ collection_namespace }}"'
+
+- name: Do file content replacements for non-default namespace or package name
+  block:
+
+    - name: Change module doc_fragments to support desired namespace and package names
+      replace:
+        path: "{{ item }}"
+        regexp: '^extends_documentation_fragment: awx.awx.auth$'
+        replace: 'extends_documentation_fragment: {{ collection_namespace }}.{{ collection_package }}.auth'
+      with_fileglob: "{{ collection_path }}/plugins/modules/tower_*.py"
+      loop_control:
+        label: "{{ item | basename }}"
+
+    - name: Change inventory file to support desired namespace and package names
+      replace:
+        path: "{{ collection_path }}/plugins/inventory/tower.py"
+        regexp: "^    NAME = 'awx.awx.tower'  # REPLACE$"
+        replace: "    NAME = '{{ collection_namespace }}.{{ collection_package }}.tower'  # REPLACE"
+  when:
+    - (collection_package != 'awx') or (collection_namespace != 'awx')
+
+- name: Template the galaxy.yml file
+  template:
+    src: "{{ collection_path }}/tools/roles/template_galaxy/templates/galaxy.yml.j2"
+    dest: "{{ collection_path }}/galaxy.yml"
+    force: true

awx_collection/tools/template_galaxy.yml

@@ -0,0 +1,12 @@
+---
+- name: Template the collection galaxy.yml
+  hosts: localhost
+  gather_facts: false
+  connection: local
+  vars:
+    collection_package: awx
+    collection_namespace: awx
+    collection_version: 0.0.1  # not for updating, pass in extra_vars
+    collection_path: "{{ playbook_dir }}/../"
+  roles:
+    - template_galaxy