From caa55f112f7ec5f42e9b5ed2db05ac5fce8c6604 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 18 Jan 2019 16:04:32 -0500 Subject: [PATCH] add robust handling of non-UTF8 when detecting inventory/playbooks --- awx/main/tests/unit/utils/ansible/__init__.py | 0 .../utils/ansible/inventories/invalid/bad | 1 + .../ansible/inventories/invalid/bad_encoding | 2 ++ .../ansible/inventories/invalid/empty.txt | 0 .../ansible/inventories/valid/executable | 0 .../utils/ansible/inventories/valid/hosts | 0 .../utils/ansible/inventories/valid/inventory | 0 .../unit/utils/ansible/playbooks/invalid/bad | 1 + .../ansible/playbooks/invalid/bad_encoding | 2 ++ .../utils/ansible/playbooks/invalid/empty | 0 .../playbooks/invalid/wrong_file_extension | 7 ++++ .../ansible/playbooks/valid/hello_world.yaml | 7 ++++ .../ansible/playbooks/valid/hello_world.yml | 7 ++++ .../utils/ansible/playbooks/valid/hosts.yml | 1 + .../utils/ansible/playbooks/valid/import.yml | 1 + .../utils/ansible/playbooks/valid/include.yml | 1 + .../utils/ansible/playbooks/valid/vault.yml | 11 +++++++ .../tests/unit/utils/ansible/test_ansible.py | 32 +++++++++++++++++++ awx/main/utils/ansible.py | 17 ++++++++-- 19 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 awx/main/tests/unit/utils/ansible/__init__.py create mode 100644 awx/main/tests/unit/utils/ansible/inventories/invalid/bad create mode 100644 awx/main/tests/unit/utils/ansible/inventories/invalid/bad_encoding create mode 100644 awx/main/tests/unit/utils/ansible/inventories/invalid/empty.txt create mode 100755 awx/main/tests/unit/utils/ansible/inventories/valid/executable create mode 100644 awx/main/tests/unit/utils/ansible/inventories/valid/hosts create mode 100644 awx/main/tests/unit/utils/ansible/inventories/valid/inventory create mode 100644 awx/main/tests/unit/utils/ansible/playbooks/invalid/bad create mode 100644 awx/main/tests/unit/utils/ansible/playbooks/invalid/bad_encoding create mode 100644 awx/main/tests/unit/utils/ansible/playbooks/invalid/empty create mode 100644 awx/main/tests/unit/utils/ansible/playbooks/invalid/wrong_file_extension create mode 100644 awx/main/tests/unit/utils/ansible/playbooks/valid/hello_world.yaml create mode 100644 awx/main/tests/unit/utils/ansible/playbooks/valid/hello_world.yml create mode 100644 awx/main/tests/unit/utils/ansible/playbooks/valid/hosts.yml create mode 100644 awx/main/tests/unit/utils/ansible/playbooks/valid/import.yml create mode 100644 awx/main/tests/unit/utils/ansible/playbooks/valid/include.yml create mode 100644 awx/main/tests/unit/utils/ansible/playbooks/valid/vault.yml create mode 100644 awx/main/tests/unit/utils/ansible/test_ansible.py diff --git a/awx/main/tests/unit/utils/ansible/__init__.py b/awx/main/tests/unit/utils/ansible/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/awx/main/tests/unit/utils/ansible/inventories/invalid/bad b/awx/main/tests/unit/utils/ansible/inventories/invalid/bad new file mode 100644 index 0000000000..ed5969ab83 --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/inventories/invalid/bad @@ -0,0 +1 @@ +$$$ diff --git a/awx/main/tests/unit/utils/ansible/inventories/invalid/bad_encoding b/awx/main/tests/unit/utils/ansible/inventories/invalid/bad_encoding new file mode 100644 index 0000000000..20fcd2da37 --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/inventories/invalid/bad_encoding @@ -0,0 +1,2 @@ +Ý +? \ No newline at end of file diff --git a/awx/main/tests/unit/utils/ansible/inventories/invalid/empty.txt b/awx/main/tests/unit/utils/ansible/inventories/invalid/empty.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/awx/main/tests/unit/utils/ansible/inventories/valid/executable b/awx/main/tests/unit/utils/ansible/inventories/valid/executable new file mode 100755 index 0000000000..e69de29bb2 diff --git a/awx/main/tests/unit/utils/ansible/inventories/valid/hosts b/awx/main/tests/unit/utils/ansible/inventories/valid/hosts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/awx/main/tests/unit/utils/ansible/inventories/valid/inventory b/awx/main/tests/unit/utils/ansible/inventories/valid/inventory new file mode 100644 index 0000000000..e69de29bb2 diff --git a/awx/main/tests/unit/utils/ansible/playbooks/invalid/bad b/awx/main/tests/unit/utils/ansible/playbooks/invalid/bad new file mode 100644 index 0000000000..5336319f5d --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/playbooks/invalid/bad @@ -0,0 +1 @@ +not a playbook diff --git a/awx/main/tests/unit/utils/ansible/playbooks/invalid/bad_encoding b/awx/main/tests/unit/utils/ansible/playbooks/invalid/bad_encoding new file mode 100644 index 0000000000..20fcd2da37 --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/playbooks/invalid/bad_encoding @@ -0,0 +1,2 @@ +Ý +? \ No newline at end of file diff --git a/awx/main/tests/unit/utils/ansible/playbooks/invalid/empty b/awx/main/tests/unit/utils/ansible/playbooks/invalid/empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/awx/main/tests/unit/utils/ansible/playbooks/invalid/wrong_file_extension b/awx/main/tests/unit/utils/ansible/playbooks/invalid/wrong_file_extension new file mode 100644 index 0000000000..20632c6155 --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/playbooks/invalid/wrong_file_extension @@ -0,0 +1,7 @@ +- name: Hello World Sample + hosts: all + tasks: + - name: Hello Message + debug: + msg: "Hello World!" + diff --git a/awx/main/tests/unit/utils/ansible/playbooks/valid/hello_world.yaml b/awx/main/tests/unit/utils/ansible/playbooks/valid/hello_world.yaml new file mode 100644 index 0000000000..20632c6155 --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/playbooks/valid/hello_world.yaml @@ -0,0 +1,7 @@ +- name: Hello World Sample + hosts: all + tasks: + - name: Hello Message + debug: + msg: "Hello World!" + diff --git a/awx/main/tests/unit/utils/ansible/playbooks/valid/hello_world.yml b/awx/main/tests/unit/utils/ansible/playbooks/valid/hello_world.yml new file mode 100644 index 0000000000..20632c6155 --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/playbooks/valid/hello_world.yml @@ -0,0 +1,7 @@ +- name: Hello World Sample + hosts: all + tasks: + - name: Hello Message + debug: + msg: "Hello World!" + diff --git a/awx/main/tests/unit/utils/ansible/playbooks/valid/hosts.yml b/awx/main/tests/unit/utils/ansible/playbooks/valid/hosts.yml new file mode 100644 index 0000000000..313de3e6a8 --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/playbooks/valid/hosts.yml @@ -0,0 +1 @@ +- hosts: all diff --git a/awx/main/tests/unit/utils/ansible/playbooks/valid/import.yml b/awx/main/tests/unit/utils/ansible/playbooks/valid/import.yml new file mode 100644 index 0000000000..fd3a5f0998 --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/playbooks/valid/import.yml @@ -0,0 +1 @@ +- import_playbook: foo diff --git a/awx/main/tests/unit/utils/ansible/playbooks/valid/include.yml b/awx/main/tests/unit/utils/ansible/playbooks/valid/include.yml new file mode 100644 index 0000000000..f2286d3610 --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/playbooks/valid/include.yml @@ -0,0 +1 @@ +- include: foo diff --git a/awx/main/tests/unit/utils/ansible/playbooks/valid/vault.yml b/awx/main/tests/unit/utils/ansible/playbooks/valid/vault.yml new file mode 100644 index 0000000000..aa392a91d7 --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/playbooks/valid/vault.yml @@ -0,0 +1,11 @@ +$ANSIBLE_VAULT;1.1;AES256 +64313966653262616130386430653233326161623364386235636436333430646161323830336663 +6633613437653635626162386338613338646231396363660a376537373331356239623435353365 +31633039363639376166633538336335383062316461633439346630363135316266613766393864 +3363343634636535650a346231653233626362323135383164636634343534333466363139633436 +66366132656364316161336538613933666537666361356662306631653235323936363764613338 +65653833326661323935373535396164373132393165383633643432306432373463376461613165 +61376361323861373036316230343038666366336231303231303937393731616664316664373338 +36623935363262623566653238313964636435666138626336346465363562356535663033356265 +64323239616536346365306635353863623831636266633233313437396236633235373539373363 +6464303039663737333164613763306137393864356663316263 diff --git a/awx/main/tests/unit/utils/ansible/test_ansible.py b/awx/main/tests/unit/utils/ansible/test_ansible.py new file mode 100644 index 0000000000..2d4b10a24b --- /dev/null +++ b/awx/main/tests/unit/utils/ansible/test_ansible.py @@ -0,0 +1,32 @@ +import os +import os.path + +import pytest + +from awx.main.utils.ansible import could_be_playbook, could_be_inventory + +HERE, _ = os.path.split(__file__) + + +@pytest.mark.parametrize('filename', os.listdir(os.path.join(HERE, 'playbooks', 'valid'))) +def test_could_be_playbook(filename): + path = os.path.join(HERE, 'playbooks', 'valid') + assert could_be_playbook(HERE, path, filename).endswith(filename) + + +@pytest.mark.parametrize('filename', os.listdir(os.path.join(HERE, 'playbooks', 'invalid'))) +def test_is_not_playbook(filename): + path = os.path.join(HERE, 'playbooks', 'invalid') + assert could_be_playbook(HERE, path, filename) is None + + +@pytest.mark.parametrize('filename', os.listdir(os.path.join(HERE, 'inventories', 'valid'))) +def test_could_be_inventory(filename): + path = os.path.join(HERE, 'inventories', 'valid') + assert could_be_inventory(HERE, path, filename).endswith(filename) + + +@pytest.mark.parametrize('filename', os.listdir(os.path.join(HERE, 'inventories', 'invalid'))) +def test_is_not_inventory(filename): + path = os.path.join(HERE, 'inventories', 'invalid') + assert could_be_inventory(HERE, path, filename) is None diff --git a/awx/main/utils/ansible.py b/awx/main/utils/ansible.py index 3ce57a8a58..7e68d88189 100644 --- a/awx/main/utils/ansible.py +++ b/awx/main/utils/ansible.py @@ -2,6 +2,7 @@ # All Rights Reserved. # Python +import codecs import re import os from itertools import islice @@ -44,13 +45,20 @@ def could_be_playbook(project_path, dir_path, filename): # show up. matched = False try: - for n, line in enumerate(open(playbook_path)): + for n, line in enumerate(codecs.open( + playbook_path, + 'r', + encoding='utf-8', + errors='ignore' + )): if valid_playbook_re.match(line): matched = True + break # Any YAML file can also be encrypted with vault; # allow these to be used as the main playbook. elif n == 0 and line.startswith('$ANSIBLE_VAULT;'): matched = True + break except IOError: return None if not matched: @@ -77,7 +85,12 @@ def could_be_inventory(project_path, dir_path, filename): # Ansible inventory mainly try: # only read through first 10 lines for performance - with open(inventory_path) as inv_file: + with codecs.open( + inventory_path, + 'r', + encoding='utf-8', + errors='ignore' + ) as inv_file: for line in islice(inv_file, 10): if not valid_inventory_re.match(line): return None