tests: introduce slow integration tests

Signed-off-by: Fernando Fernandez Mancera <ffmancera@riseup.net>

.travis.yml

@@ -11,6 +11,8 @@ env:
           use_coveralls=false
         - CONTAINER_IMAGE=nmstate/fedora-nmstate-dev
           testflags="--test-type integ --pytest-args='-x'"
+        - CONTAINER_IMAGE=nmstate/centos8-nmstate-dev
+          testflags="--test-type integ_slow --pytest-args='-x'"
         - CONTAINER_IMAGE=nmstate/centos8-nmstate-dev-nm-1.22-git
           testflags="--test-type integ --pytest-args='-x'"
         - CONTAINER_IMAGE=nmstate/centos8-nmstate-dev
@@ -23,11 +25,14 @@ env:
           testflags="--test-type unit_py38"
 
 matrix:
+    fast_finish: true
     allow_failures:
         - env: CONTAINER_IMAGE=nmstate/fedora-nmstate-dev
                testflags="--test-type integ --pytest-args='-x'"
         - env: CONTAINER_IMAGE=nmstate/centos8-nmstate-dev-nm-1.22-git
                testflags="--test-type integ --pytest-args='-x'"
+        - env: CONTAINER_IMAGE=nmstate/centos8-nmstate-dev
+               testflags="--test-type integ_slow --pytest-args='-x'"
 
 addons:
     apt:

automation/README.md

@@ -28,10 +28,15 @@ using 'nmstate/fedora-nmstate-dev' docker image.
 You may change the test type by specifying the `--test-type` flag, for example:
 
  * `./automation/run-tests.sh --test-type integ --el8`:
-   Integration tests using 'nmstate/centos8-nmstate-dev' docker image.
+   Integration tests (without slow test cases) using
+   'nmstate/centos8-nmstate-dev' docker image.
 
  * `./automation/run-tests.sh --test-type integ`:
-   Integration tests using 'nmstate/fedora-nmstate-dev' docker image.
+   Integration tests (without slow test cases) using
+   'nmstate/fedora-nmstate-dev' docker image.
+
+ * `./automation/run-tests.sh --test-type integ_slow`:
+   Integration slow test cases using `nmstate/fedora-nmstate-dev` docker image.
 
 For a full list of command-line flags, run `./automation/run-tests.sh --help`.
 

automation/run-tests.sh

@@ -13,10 +13,20 @@ TEST_TYPE_LINT="lint"
 TEST_TYPE_UNIT_PY36="unit_py36"
 TEST_TYPE_UNIT_PY38="unit_py38"
 TEST_TYPE_INTEG="integ"
+TEST_TYPE_INTEG_SLOW="integ_slow"
 
 FEDORA_IMAGE_DEV="nmstate/fedora-nmstate-dev"
 CENTOS_IMAGE_DEV="nmstate/centos8-nmstate-dev"
 
+PYTEST_OPTIONS="--verbose --verbose \
+        --log-level=DEBUG \
+        --log-date-format='%Y-%m-%d %H:%M:%S' \
+        --log-format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s' \
+        --durations=5 \
+        --cov /usr/lib/python*/site-packages/libnmstate \
+        --cov /usr/lib/python*/site-packages/nmstatectl \
+        --cov-report=term"
+
 : ${CONTAINER_CMD:=docker}
 
 test -t 1 && USE_TTY="-t"
@@ -109,15 +119,20 @@ function run_tests {
         container_exec "
           cd $CONTAINER_WORKSPACE &&
           pytest \
-            --verbose --verbose \
-            --log-level=DEBUG \
-            --log-date-format='%Y-%m-%d %H:%M:%S' \
-            --log-format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s' \
-            --durations=5 \
-            --cov /usr/lib/python*/site-packages/libnmstate \
-            --cov /usr/lib/python*/site-packages/nmstatectl \
+            $PYTEST_OPTIONS \
             --cov-report=html:htmlcov-integ \
-            --cov-report=term \
+            tests/integration \
+            ${nmstate_pytest_extra_args}"
+    fi
+
+    if [ $TEST_TYPE == $TEST_TYPE_ALL ] || \
+       [ $TEST_TYPE == $TEST_TYPE_INTEG_SLOW ];then
+        container_exec "
+          cd $CONTAINER_WORKSPACE &&
+          pytest \
+            $PYTEST_OPTIONS \
+            --cov-report=html:htmlcov-integ_slow \
+            -m slow --runslow \
             tests/integration \
             ${nmstate_pytest_extra_args}"
     fi
@@ -238,6 +253,7 @@ while true; do
         echo "     * $TEST_TYPE_FORMAT"
         echo "     * $TEST_TYPE_LINT"
         echo "     * $TEST_TYPE_INTEG"
+        echo "     * $TEST_TYPE_INTEG_SLOW"
         echo "     * $TEST_TYPE_UNIT_PY36"
         echo "     * $TEST_TYPE_UNIT_PY38"
         echo -n "--customize allows to specify a command to customize the "

tests/integration/conftest.py

@@ -33,6 +33,22 @@ OS: {osname}
 """
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--runslow", action="store_true", default=False, help="run slow tests"
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--runslow"):
+        # --runslow is given in cli: do not skip slow tests
+        return
+    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+    for item in items:
+        if "slow" in item.keywords:
+            item.add_marker(skip_slow)
+
+
 @pytest.fixture(scope="session", autouse=True)
 def logging_setup():
     logging.basicConfig(