2020-03-24 14:01:12 +03:00
variables :
2020-03-24 15:04:08 +03:00
GIT_DEPTH : 100
2020-03-24 14:01:12 +03:00
2020-03-06 18:38:36 +03:00
stages :
2020-06-02 18:28:57 +03:00
- containers
ci: Reduce number of stages
Right now we're dividing the jobs into three stages: prebuild, which
includes DCO checking as well as building artifacts such as the
website, and native_build/cross_build, which do exactly what you'd
expect based on their names.
This organization is nice from the logical point of view, but results
in poor utilization of the available CI resources: in particular, the
fact that cross_build jobs can only start after all native_build jobs
have finished means that if even a single one of the latter takes a
bit longer the pipeline will stall, and with native builds taking
anywhere from less than 10 minutes to more than 20, this happens all
the time.
Building artifacts in a separate pipeline stage also doesn't have any
advantages, and only delays further stages by a couple of minutes.
The only job that really makes sense in its own stage is the DCO
check, because it's extremely fast (less than 1 minute) and, if that
fails, we can avoid kicking off all other jobs.
Reducing the number of stages results in significant speedups:
specifically, going from three stages to two stages reduces the
overall completion time for a full CI pipeline from ~45 minutes[1]
to ~30 minutes[2].
[1] https://gitlab.com/abologna/libvirt/-/pipelines/154751893
[2] https://gitlab.com/abologna/libvirt/-/pipelines/154771173
Signed-off-by: Andrea Bolognani <abologna@redhat.com>
Reviewed-by: Daniel P. Berrangé <berrange@redhat.com>
2020-06-10 13:11:30 +03:00
- builds
2021-10-05 12:02:24 +03:00
- integration_tests
2020-07-28 12:05:16 +03:00
- sanity_checks
2020-03-06 18:38:36 +03:00
2020-03-27 18:40:05 +03:00
.script_variables : &script_variables |
gitlab: Enable improved ccache usage
Setting CC="ccache cc" works in most cases, but sometimes it will
break the build: in particular, we have experienced issues in the
past with that approach when using cgo to build our Go bindings.
A more robust approach is to have a directory containing symlinks
from the compiler name to the ccache binary: in that case, ccache
itself will invoke the compiler, and the build system will be none
the wiser.
Since libvirt-ci commit 2563aebb6c5c, container images contain a
suitable symlink directory, so all that's needed to enable the new
approach is to add this directory to $PATH.
Since we're touching this anyway, we make a few more changes:
$CCACHE_DIR is no longer created manually, because ccache will
take care of creating it for us if it doesn't already exist; the
ccache setup is moved out of the job template and into
script_variables, removing unnecessary duplication; a limit is
set on the size of the cache (500 MB, which is twice the amount
used by a fresh build on my Fedora 31 machine).
Signed-off-by: Andrea Bolognani <abologna@redhat.com>
Reviewed-by: Daniel P. Berrangé <berrange@redhat.com>
2020-03-30 19:29:06 +03:00
export CCACHE_BASEDIR="$(pwd)"
export CCACHE_DIR="$CCACHE_BASEDIR/ccache"
export CCACHE_MAXSIZE="500M"
export PATH="$CCACHE_WRAPPERSDIR:$PATH"
2021-05-05 16:44:47 +03:00
export VIR_TEST_VERBOSE="1"
export VIR_TEST_DEBUG="1"
2020-03-06 18:38:36 +03:00
2021-10-05 12:02:24 +03:00
include :
- '/ci/gitlab.yml'
- '/ci/integration.yml'
2020-06-02 18:28:57 +03:00
2021-01-13 19:45:06 +03:00
.native_build_job :
2021-09-09 16:49:01 +03:00
extends : .gitlab_native_build_job
2020-03-25 18:01:43 +03:00
cache :
paths :
- ccache/
key : "$CI_JOB_NAME"
before_script :
2020-03-27 18:40:05 +03:00
- *script_variables
2021-11-23 19:00:17 +03:00
- cat /packages.txt
2020-03-06 19:29:03 +03:00
script :
2021-09-09 16:47:48 +03:00
- meson setup build --werror $MESON_ARGS || (cat build/meson-logs/meson-log.txt && exit 1)
2021-05-10 20:14:12 +03:00
- meson dist -C build --no-tests
2020-10-08 16:36:07 +03:00
- if test -x /usr/bin/rpmbuild && test "$RPM" != "skip";
then
2021-10-05 12:01:34 +03:00
rpmbuild --clean --nodeps --define "_topdir $PWD/rpmbuild/" -ta build/meson-dist/libvirt-*.tar.xz;
mv rpmbuild/RPMS/x86_64/ libvirt-rpms/;
2021-05-10 20:14:12 +03:00
else
meson compile -C build;
meson test -C build --no-suite syntax-check --print-errorlogs;
2020-10-08 16:36:07 +03:00
fi
2020-03-06 19:29:03 +03:00
2021-01-13 19:45:06 +03:00
.cross_build_job :
2021-09-09 16:49:01 +03:00
extends : .gitlab_cross_build_job
2020-03-25 18:01:43 +03:00
cache :
paths :
- ccache/
key : "$CI_JOB_NAME"
before_script :
2020-03-27 18:40:05 +03:00
- *script_variables
2021-11-23 19:00:17 +03:00
- cat /packages.txt
2019-01-25 20:38:53 +03:00
script :
2021-05-10 20:20:30 +03:00
- meson setup build --werror $MESON_OPTS || (cat build/meson-logs/meson-log.txt && exit 1)
- meson compile -C build
2021-05-10 20:14:12 +03:00
- if test "$CROSS" = "i686" ; then meson test -C build --no-suite syntax-check --print-errorlogs ; fi
2019-01-25 20:38:53 +03:00
2020-03-06 17:56:42 +03:00
# This artifact published by this job is downloaded by libvirt.org to
# be deployed to the web root:
# https://gitlab.com/libvirt/libvirt/-/jobs/artifacts/master/download?job=website
website :
ci: Reduce number of stages
Right now we're dividing the jobs into three stages: prebuild, which
includes DCO checking as well as building artifacts such as the
website, and native_build/cross_build, which do exactly what you'd
expect based on their names.
This organization is nice from the logical point of view, but results
in poor utilization of the available CI resources: in particular, the
fact that cross_build jobs can only start after all native_build jobs
have finished means that if even a single one of the latter takes a
bit longer the pipeline will stall, and with native builds taking
anywhere from less than 10 minutes to more than 20, this happens all
the time.
Building artifacts in a separate pipeline stage also doesn't have any
advantages, and only delays further stages by a couple of minutes.
The only job that really makes sense in its own stage is the DCO
check, because it's extremely fast (less than 1 minute) and, if that
fails, we can avoid kicking off all other jobs.
Reducing the number of stages results in significant speedups:
specifically, going from three stages to two stages reduces the
overall completion time for a full CI pipeline from ~45 minutes[1]
to ~30 minutes[2].
[1] https://gitlab.com/abologna/libvirt/-/pipelines/154751893
[2] https://gitlab.com/abologna/libvirt/-/pipelines/154771173
Signed-off-by: Andrea Bolognani <abologna@redhat.com>
Reviewed-by: Daniel P. Berrangé <berrange@redhat.com>
2020-06-10 13:11:30 +03:00
stage : builds
2022-02-14 19:44:12 +03:00
image : $CI_REGISTRY_IMAGE/ci-almalinux-8:latest
2020-07-28 12:15:13 +03:00
needs :
2022-05-27 14:47:07 +03:00
- job : x86_64-almalinux-8-container
optional : true
2020-03-27 18:40:05 +03:00
before_script :
- *script_variables
2020-03-06 17:56:42 +03:00
script :
2021-05-10 20:20:30 +03:00
- meson setup build --werror --prefix=$(pwd)/vroot || (cat build/meson-logs/meson-log.txt && exit 1)
2020-08-03 09:53:00 +03:00
- ninja -C build install-web
2020-03-06 17:56:42 +03:00
- mv vroot/share/doc/libvirt/html/ website
artifacts :
expose_as : 'Website'
name : 'website'
when : on_success
expire_in : 30 days
paths :
- website
2020-03-24 17:42:14 +03:00
2020-03-26 15:07:34 +03:00
codestyle :
2021-05-12 20:27:56 +03:00
stage : sanity_checks
2022-04-12 18:29:05 +03:00
image : $CI_REGISTRY_IMAGE/ci-opensuse-leap-153:latest
2020-07-28 12:15:13 +03:00
needs :
2022-05-27 14:47:07 +03:00
- job : x86_64-opensuse-leap-153-container
optional : true
2020-03-27 18:40:05 +03:00
before_script :
- *script_variables
2020-03-26 15:07:34 +03:00
script :
2021-05-10 20:20:30 +03:00
- meson setup build --werror || (cat build/meson-logs/meson-log.txt && exit 1)
2020-08-03 09:53:00 +03:00
- ninja -C build libvirt-pot-dep
2021-05-10 19:57:50 +03:00
- meson test -C build --suite syntax-check --no-rebuild --print-errorlogs
2020-03-26 15:07:34 +03:00
2020-03-24 17:42:14 +03:00
# This artifact published by this job is downloaded to push to Weblate
# for translation usage:
# https://gitlab.com/libvirt/libvirt/-/jobs/artifacts/master/download?job=potfile
potfile :
ci: Reduce number of stages
Right now we're dividing the jobs into three stages: prebuild, which
includes DCO checking as well as building artifacts such as the
website, and native_build/cross_build, which do exactly what you'd
expect based on their names.
This organization is nice from the logical point of view, but results
in poor utilization of the available CI resources: in particular, the
fact that cross_build jobs can only start after all native_build jobs
have finished means that if even a single one of the latter takes a
bit longer the pipeline will stall, and with native builds taking
anywhere from less than 10 minutes to more than 20, this happens all
the time.
Building artifacts in a separate pipeline stage also doesn't have any
advantages, and only delays further stages by a couple of minutes.
The only job that really makes sense in its own stage is the DCO
check, because it's extremely fast (less than 1 minute) and, if that
fails, we can avoid kicking off all other jobs.
Reducing the number of stages results in significant speedups:
specifically, going from three stages to two stages reduces the
overall completion time for a full CI pipeline from ~45 minutes[1]
to ~30 minutes[2].
[1] https://gitlab.com/abologna/libvirt/-/pipelines/154751893
[2] https://gitlab.com/abologna/libvirt/-/pipelines/154771173
Signed-off-by: Andrea Bolognani <abologna@redhat.com>
Reviewed-by: Daniel P. Berrangé <berrange@redhat.com>
2020-06-10 13:11:30 +03:00
stage : builds
2022-02-14 19:44:12 +03:00
image : $CI_REGISTRY_IMAGE/ci-almalinux-8:latest
2020-07-28 12:15:13 +03:00
needs :
2022-05-27 14:47:07 +03:00
- job : x86_64-almalinux-8-container
optional : true
2021-01-14 13:20:36 +03:00
rules :
- if : "$CI_COMMIT_BRANCH == 'master'"
2020-03-27 18:40:05 +03:00
before_script :
- *script_variables
2020-03-24 17:42:14 +03:00
script :
2021-05-10 20:20:30 +03:00
- meson setup build --werror || (cat build/meson-logs/meson-log.txt && exit 1)
2020-08-03 09:53:00 +03:00
- ninja -C build libvirt-pot-dep
- ninja -C build libvirt-pot
2020-06-04 16:26:39 +03:00
- cp po/libvirt.pot libvirt.pot
2020-03-24 17:42:14 +03:00
artifacts :
expose_as : 'Potfile'
name : 'potfile'
when : on_success
expire_in : 30 days
paths :
- libvirt.pot
2020-03-26 13:46:47 +03:00
2020-11-12 16:56:25 +03:00
# Coverity job that is run only by schedules
coverity :
2022-02-14 19:44:12 +03:00
image : $CI_REGISTRY_IMAGE/ci-almalinux-8:latest
2020-11-12 16:56:25 +03:00
needs :
2022-05-27 14:47:07 +03:00
- job : x86_64-almalinux-8-container
optional : true
2020-11-12 16:56:25 +03:00
stage : builds
script :
- curl https://scan.coverity.com/download/linux64 --form project=$COVERITY_SCAN_PROJECT_NAME --form token=$COVERITY_SCAN_TOKEN -o /tmp/cov-analysis-linux64.tgz
- tar xfz /tmp/cov-analysis-linux64.tgz
2021-05-10 20:20:30 +03:00
- meson setup build --werror || (cat build/meson-logs/meson-log.txt && exit 1)
- cov-analysis-linux64-*/bin/cov-build --dir cov-int meson compile -C build
2020-11-12 16:56:25 +03:00
- tar cfz cov-int.tar.gz cov-int
- curl https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME --form token=$COVERITY_SCAN_TOKEN --form email=$GITLAB_USER_EMAIL --form file=@cov-int.tar.gz --form version="$(git describe --tags)" --form description="$(git describe --tags) / $CI_COMMIT_TITLE / $CI_COMMIT_REF_NAME:$CI_PIPELINE_ID"
2021-01-14 13:20:36 +03:00
rules :
- if : "$CI_PIPELINE_SOURCE == 'schedule' && $COVERITY_SCAN_PROJECT_NAME && $COVERITY_SCAN_TOKEN"