tests, integ: introduce integ_tier1 and integ_tier2 integration tests

This patch is introducing integ_tier1 and integ_tier2 tests and using it
in travis CI.  This will allow us to run integration test quickly as
both jobs, integ_tier1 and integ_tier2 are going to run at the same
time.

Signed-off-by: Fernando Fernandez Mancera <ffmancera@riseup.net>

.travis.yml

@@ -6,7 +6,12 @@ env:
         - use_coveralls=true
     matrix:
         - CONTAINER_IMAGE=docker.io/nmstate/centos8-nmstate-dev
-          testflags="--test-type integ --pytest-args='-x'
+          testflags="--test-type integ_tier1 --pytest-args='-x'
+              --copr networkmanager/NetworkManager-1.22-git"
+          use_codecov=true
+          use_coveralls=false
+        - CONTAINER_IMAGE=docker.io/nmstate/centos8-nmstate-dev
+          testflags="--test-type integ_tier2 --pytest-args='-x'
               --copr networkmanager/NetworkManager-1.22-git"
           use_codecov=true
           use_coveralls=false

automation/README.md

@@ -35,6 +35,14 @@ You may change the test type by specifying the `--test-type` flag, for example:
    Integration tests (without slow test cases) using
    'nmstate/fedora-nmstate-dev' container image.
 
+ * `./automation/run-tests.sh --test-type integ_tier1`:
+   Integration tier1 test cases using `nmstate/fedora-nmstate-dev` container
+   image.
+
+ * `./automation/run-tests.sh --test-type integ_tier2`:
+   Integration tier2 test cases using `nmstate/fedora-nmstate-dev` container
+   image.
+
  * `./automation/run-tests.sh --test-type integ_slow`:
    Integration slow test cases using `nmstate/fedora-nmstate-dev` container
    image.

automation/run-tests.sh

@@ -13,6 +13,8 @@ TEST_TYPE_LINT="lint"
 TEST_TYPE_UNIT_PY36="unit_py36"
 TEST_TYPE_UNIT_PY38="unit_py38"
 TEST_TYPE_INTEG="integ"
+TEST_TYPE_INTEG_TIER1="integ_tier1"
+TEST_TYPE_INTEG_TIER2="integ_tier2"
 TEST_TYPE_INTEG_SLOW="integ_slow"
 
 FEDORA_IMAGE_DEV="docker.io/nmstate/fedora-nmstate-dev"
@@ -127,6 +129,28 @@ function run_tests {
             ${nmstate_pytest_extra_args}"
     fi
 
+    if  [ $TEST_TYPE == $TEST_TYPE_INTEG_TIER1 ];then
+        container_exec "
+          cd $CONTAINER_WORKSPACE &&
+          pytest \
+            $PYTEST_OPTIONS \
+            --cov-report=html:htmlcov-integ_tier1 \
+            -m tier1 \
+            tests/integration \
+            ${nmstate_pytest_extra_args}"
+    fi
+
+    if  [ $TEST_TYPE == $TEST_TYPE_INTEG_TIER2 ];then
+        container_exec "
+          cd $CONTAINER_WORKSPACE &&
+          pytest \
+            $PYTEST_OPTIONS \
+            --cov-report=html:htmlcov-integ_tier2 \
+            -m tier2 \
+            tests/integration \
+            ${nmstate_pytest_extra_args}"
+    fi
+
     if [ $TEST_TYPE == $TEST_TYPE_ALL ] || \
        [ $TEST_TYPE == $TEST_TYPE_INTEG_SLOW ];then
         container_exec "
@@ -288,6 +312,8 @@ while true; do
         echo "     * $TEST_TYPE_FORMAT"
         echo "     * $TEST_TYPE_LINT"
         echo "     * $TEST_TYPE_INTEG"
+        echo "     * $TEST_TYPE_INTEG_TIER1"
+        echo "     * $TEST_TYPE_INTEG_TIER2"
         echo "     * $TEST_TYPE_INTEG_SLOW"
         echo "     * $TEST_TYPE_UNIT_PY36"
         echo "     * $TEST_TYPE_UNIT_PY38"

tests/integration/conftest.py

@@ -44,15 +44,25 @@ def pytest_addoption(parser):
 
 
 def pytest_collection_modifyitems(config, items):
-    if config.getoption("--runslow"):
-        # --runslow is given in cli: do not skip slow tests
-        return
+    if not config.getoption("--runslow"):
+        # --runslow is not in cli: skip slow tests
+        _mark_skip_slow_tests(items)
+    _mark_tier2_tests(items)
+
+
+def _mark_skip_slow_tests(items):
     skip_slow = pytest.mark.skip(reason="need --runslow option to run")
     for item in items:
         if "slow" in item.keywords:
             item.add_marker(skip_slow)
 
 
+def _mark_tier2_tests(items):
+    for item in items:
+        if "tier1" not in item.keywords:
+            item.add_marker(pytest.mark.tier2)
+
+
 @pytest.fixture(scope="session", autouse=True)
 def test_env_setup():
     _logging_setup()