From f6c6f033d9963b218d22b6000fbcd9bb98310a8d Mon Sep 17 00:00:00 2001 From: Cole Robinson Date: Fri, 17 Jul 2020 15:59:11 -0400 Subject: [PATCH] tests: Move checkprops.py magic skipping to pytest We want to skip these tests if previous tests failed or were skipped, but current impl depends on unittest specifics. Move it to pytest Signed-off-by: Cole Robinson --- tests/conftest.py | 24 +++++++-- tests/test_checkprops.py | 111 +++++++++++++++++---------------------- tests/utils.py | 1 + 3 files changed, 67 insertions(+), 69 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2cb0a9495..3d1ca7d15 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,9 @@ import os import pytest +import tests +from tests.utils import TESTCONFIG + def pytest_addoption(parser): parser.addoption("--uitests", action="store_true", default=False, @@ -70,18 +73,29 @@ def pytest_collection_modifyitems(config, items): if not find_items("test_urls.py"): # Don't setup urlfetcher mocking for test_urls.py # All other tests need it - import tests.urlfetcher_mock - tests.urlfetcher_mock.setup_mock() + from tests import urlfetcher_mock + urlfetcher_mock.setup_mock() if find_items("test_inject.py"): if not config.getoption("--capture") == "no": pytest.fail("test_inject.py requires `pytest --capture=no`") -def pytest_configure(config): - import tests - from tests.utils import TESTCONFIG +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + """ + Hooks into test reporting so we can track if any tests didn't + pass, so we can skip checkprops checks which require everything to pass. + """ + ignore = item + ignore = call + outcome = yield + testreport = outcome.get_result() + if not testreport.passed: + TESTCONFIG.skip_checkprops = True + +def pytest_configure(config): TESTCONFIG.url_iso_only = config.getoption("--urls-iso-only") TESTCONFIG.url_only = config.getoption("--urls-url-only") TESTCONFIG.url_skip_libosinfo = config.getoption("--urls-skip-libosinfo") diff --git a/tests/test_checkprops.py b/tests/test_checkprops.py index 1ccb254b2..06c71c514 100644 --- a/tests/test_checkprops.py +++ b/tests/test_checkprops.py @@ -2,78 +2,61 @@ # See the COPYING file in the top-level directory. import traceback -import unittest + +import pytest + +import tests.utils import virtinst -from virtinst import log -_do_skip = None +def _skipIfTestsFailed(): + if tests.utils.TESTCONFIG.skip_checkprops: + pytest.skip("Other tests failed or were skipped, don't do prop checks") -class CheckPropsTest(unittest.TestCase): - maxDiff = None +def testCheckXMLBuilderProps(): + """ + If a certain environment variable is set, XMLBuilder tracks + every property registered and every one of those that is + actually altered. The test suite sets that env variable. + If no tests failed or were skipped, we check to ensure the + test suite is tickling every XML property + """ + _skipIfTestsFailed() - def _skipIfTestsFailed(self): - # pylint: disable=protected-access - # Access to protected member, needed to unittest stuff - global _do_skip - if _do_skip is None: - _do_skip = False - try: - # Accessing an internal detail of unittest, but it's only - # to prevent incorrect output in the case that other tests - # failed or were skipped, which can give a false positive here - result = self._outcome.result - _do_skip = bool( - result.errors or result.failures or result.skipped) - except Exception: - log.debug("unittest skip hack failed", exc_info=True) + # pylint: disable=protected-access + fail = [p for p in virtinst.xmlbuilder._allprops + if p not in virtinst.xmlbuilder._seenprops] + msg = None + try: + assert fail == [] + except AssertionError: + msg = "".join(traceback.format_exc()) + "\n\n" + msg += ("This means that there are XML properties that are\n" + "untested in the test suite. This could be caused\n" + "by a previous test suite failure, or if you added\n" + "a new property and didn't extend the test suite.\n" + "Look into extending test_cli.py and/or test_xmlparse.py.") - if _do_skip: - self.skipTest("skipping as other tests failed/skipped") + if msg: + pytest.fail(msg) - def testCheckXMLBuilderProps(self): - """ - If a certain environment variable is set, XMLBuilder tracks - every property registered and every one of those that is - actually altered. The test suite sets that env variable. - If no tests failed or were skipped, we check to ensure the - test suite is tickling every XML property - """ - self._skipIfTestsFailed() - # pylint: disable=protected-access - fail = [p for p in virtinst.xmlbuilder._allprops - if p not in virtinst.xmlbuilder._seenprops] - msg = None - try: - self.assertEqual([], fail) - except AssertionError: - msg = "".join(traceback.format_exc()) + "\n\n" - msg += ("This means that there are XML properties that are\n" - "untested in the test suite. This could be caused\n" - "by a previous test suite failure, or if you added\n" - "a new property and didn't extend the test suite.\n" - "Look into extending test_cli.py and/or test_xmlparse.py.") +def testCheckCLISuboptions(): + """ + Track which command line suboptions and aliases we actually hit with + the test suite. + """ + _skipIfTestsFailed() - if msg: - self.fail(msg) - - def testCheckCLISuboptions(self): - """ - Track which command line suboptions and aliases we actually hit with - the test suite. - """ - self._skipIfTestsFailed() - - # pylint: disable=protected-access - from virtinst import cli - unchecked = cli._SuboptChecker.get_unseen() - if unchecked: - msg = "\n\n" - msg += "\n".join(sorted(a for a in unchecked)) + "\n\n" - msg += ("These command line arguments or aliases are not checked\n" - "in the test suite. Please test them.\n" - "Total unchecked arguments: %s" % len(unchecked)) - self.fail(msg) + # pylint: disable=protected-access + from virtinst import cli + unchecked = cli._SuboptChecker.get_unseen() + if unchecked: + msg = "\n\n" + msg += "\n".join(sorted(a for a in unchecked)) + "\n\n" + msg += ("These command line arguments or aliases are not checked\n" + "in the test suite. Please test them.\n" + "Total unchecked arguments: %s" % len(unchecked)) + pytest.fail(msg) diff --git a/tests/utils.py b/tests/utils.py index 5e9673dc7..46b46b4b9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -25,6 +25,7 @@ class _TestConfig(object): def __init__(self): self.regenerate_output = False self.debug = False + self.skip_checkprops = False self.url_only = False self.url_iso_only = False