1
0
mirror of https://github.com/samba-team/samba.git synced 2024-12-21 09:34:19 +03:00
samba-mirror/.gitlab-ci-main.yml
Douglas Bagnall 929f4d0ca5 bootstrap: we don't need python-crypt-r anymore
Recent commits made pyglue wrap the same crypt[_r] that dsdb modules
use, which removes an extenal dependency.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=15756

Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Andreas Schneider <asn@samba.org>
2024-12-20 07:04:31 +00:00

683 lines
21 KiB
YAML

# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
# Stages explained
#
# images: Build the images with the bootstrap script
# build_first: Build a few things first to find silly errors (fast job)
# (don't pay for 35 machines until something compiles)
# build: The main parallel job
# (keep these to 1hour as we are billed per hour)
# test_only: Tests using the build from prior stages, these typically
# have an explicit dependency defined to a specific build job,
# which means that start as soon as the build job finished.
# test_private: Like test_only, but running on private runners
# report: Code coverage reporting
stages:
- images
- build_first
- build
- test_only
- test_private
- report
variables:
# We want to be resilient to runner failures
ARTIFACT_DOWNLOAD_ATTEMPTS: "3"
EXECUTOR_JOB_SECTION_ATTEMPTS: "3"
GET_SOURCES_ATTEMPTS: "3"
RESTORE_CACHE_ATTEMPTS: "3"
#
GIT_STRATEGY: fetch
GIT_DEPTH: "3"
#
# Use GZip by default, it is fast and is good enough. Other options include --xz
SAMBA_TESTBASE_TAR_OPTIONS: -z
#
# we run autobuild.py inside a samba CI docker image located on gitlab's registry
# overwrite this variable if you want use your own image registry.
#
# Or better ask for access to the shared development repository, see
# https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access
#
SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba
#
# Set this to the contents of bootstrap/sha1sum.txt
# which is generated by bootstrap/template.py --render
#
SAMBA_CI_CONTAINER_TAG: a234da56a3f4ea96265e6c8a308051be3478ab22
#
# We use the ubuntu2204 image as default as
# it matches what we have on atb-devel-224
#
SAMBA_CI_CONTAINER_IMAGE: ubuntu2204
#
# The following images are available
# Please see the samba-o3 sections at the end of this file!
# We should run that for each available image
#
SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ubuntu2004
SAMBA_CI_CONTAINER_IMAGE_ubuntu2204: ubuntu2204
SAMBA_CI_CONTAINER_IMAGE_debian11: debian11
SAMBA_CI_CONTAINER_IMAGE_debian11_32bit: debian11-32bit
SAMBA_CI_CONTAINER_IMAGE_debian12: debian12
SAMBA_CI_CONTAINER_IMAGE_opensuse155: opensuse155
SAMBA_CI_CONTAINER_IMAGE_rocky8: rocky8
SAMBA_CI_CONTAINER_IMAGE_centos9s: centos9s
SAMBA_CI_CONTAINER_IMAGE_fedora41: fedora41
include:
# The image creation details are specified in a separate file
# See bootstrap/README.md for details
- 'bootstrap/.gitlab-ci.yml'
.shared_runner_build_image:
extends: .shared_runner_build
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE}
image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_JOB_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
.shared_template:
extends: .shared_runner_build_image
# All Samba jobs are interruptible, this avoids burning CPU when a
# newer branch is pushed.
interruptible: true
timeout: 2h
# Otherwise we run twice, once on push and once on MR
# https://forum.gitlab.com/t/new-rules-syntax-and-detached-pipelines/37292
rules:
- if: $CI_MERGE_REQUEST_ID
when: never
- when: on_success
variables:
AUTOBUILD_JOB_NAME: $CI_JOB_NAME
stage: build
cache:
key: ccache.${CI_JOB_NAME}.${SAMBA_CI_JOB_IMAGE}.${SAMBA_CI_FLAVOR}
paths:
- ccache
# This is overridden in many cases, but ensures none of the other
# main jobs start until and unless this build finishes. However
# this also ensures we do not download artifacts from any build
# unless we specifically depend on it, saving bandwidth
needs:
- job: samba-def-build
artifacts: false
before_script:
- uname -a
- ls -l /sys/module/
- ls -l /sys/kernel/security/
- if [ -e /sys/kernel/security/lsm ]; then cat /sys/kernel/security/lsm ; echo; fi
- if [ -e /proc/config.gz ]; then sudo zcat /proc/config.gz; echo; fi
- lsb_release -a
- cat /etc/os-release
- id
- cat /proc/self/status
- lscpu
- cat /proc/cpuinfo
- mount
- df -h
- cat /proc/swaps
- free -h
# ld will fail if coverage enabled, force link ld to ld.bfd
- if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi
# See bootstrap/.gitlab-ci.yml how to generate a new image
- echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]"
- echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]"
- echo "SAMBA_CI_JOB_IMAGE[${SAMBA_CI_JOB_IMAGE}]"
- echo "CI_JOB_IMAGE[${CI_JOB_IMAGE}]"
- bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt
- diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt
- echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt
- diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt
- diff -u bootstrap/sha1sum.txt /sha1sum.txt
- echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt
- export CCACHE_BASEDIR="${PWD}"
- export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR"
- export CC="ccache cc"
- export CXX="ccache c++"
- ccache -z -M 500M
- ccache -s
# We are already running .gitlab-ci directives from this repo, remove additional checks that break our CI
- git config --global --add safe.directory '*'
after_script:
- mount
- df -h
- cat /proc/swaps
- free -h
- CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c
artifacts:
expire_in: 1 week
paths:
- "*.stdout"
- "*.stderr"
- "*.info"
- public
- system-info.txt
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
- api_failure
- runner_unsupported
- stale_schedule
- archived_failure
- scheduler_failure
- data_integrity_failure
script:
# gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
# autobuild name, which means we can define a default template that runs most autobuild jobs
- script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase
# Ensure when adding a new job below that you also add it to
# the dependencies for 'pages' below for the code coverage page
# generation.
others:
extends: .shared_template
script:
- script/autobuild.py pidl $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/pidl
- script/autobuild.py replace $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/replace
- script/autobuild.py talloc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/talloc
- script/autobuild.py tdb $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/tdb
- script/autobuild.py tevent $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/tevent
- script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/samba-xc
- script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/docs-xml
.shared_template_build_only:
extends: .shared_template
timeout: 2h
needs:
artifacts:
expire_in: 1 week
paths:
- "*.stdout"
- "*.stderr"
- "*.info"
- system-info.txt
- samba-testbase.tar
script:
# gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
# autobuild name, which means we can define a default template that runs most autobuild jobs
- script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase
# On success we need to pack everything into an artifacts file
# which needs to be in the git checkout.
# As tar doesn't handle hardlink of read-only files,
# we remember the acls and add write permissions
# before creating the archive. The consumer will apply
# the acls again.
- cp -a /sha1sum.txt /builds/samba-testbase/image-sha1sum.txt
- cp -a /tmp/commit.txt /builds/samba-testbase/commit.txt
- ln -s /builds/samba-testbase/${AUTOBUILD_JOB_NAME}/ /builds/samba-testbase/build_subdir_link
- pushd /builds && getfacl -R samba-testbase > samba-testbase.acl.dump && popd
- chmod -R +w /builds/samba-testbase
- mv /builds/samba-testbase.acl.dump /builds/samba-testbase/
- tar $SAMBA_TESTBASE_TAR_OPTIONS -cf samba-testbase.tar /builds/samba-testbase
- ls -la samba-testbase.tar
- sha1sum samba-testbase.tar
.shared_template_test_only:
extends:
- .shared_template
- .shared_runner_test
stage: test_only
script:
# Print the Kerberos version to check we ended up with the right one
# in the runner. We do not have configure output to recognize it
# otherwise.
- if [ -x "$(command -v krb5-config)" ]; then krb5-config --version; fi
# We unpack the artifacts file created by the .shared_template_build_only
# run we depend on
- ls -la samba-testbase.tar
- sha1sum samba-testbase.tar
- tar $SAMBA_TESTBASE_TAR_OPTIONS -xf samba-testbase.tar -C /
- diff -u /builds/samba-testbase/image-sha1sum.txt /sha1sum.txt
- diff -u /builds/samba-testbase/commit.txt /tmp/commit.txt
- mv /builds/samba-testbase/samba-testbase.acl.dump /builds/samba-testbase.acl.dump
- pushd /builds && setfacl --restore=/builds/samba-testbase.acl.dump && popd
- ls -la /builds/samba-testbase/
- ls -la /builds/samba-testbase/build_subdir_link
- ls -la /builds/samba-testbase/build_subdir_link/
- if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then find /builds/samba-testbase/build_subdir_link/ -type d -printf "'%p'\n" | xargs chmod u+w; fi
- ls -la /builds/samba-testbase/build_subdir_link/
# gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
# autobuild name, which means we can define a default template that runs most autobuild jobs
- script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase
samba-def-build:
extends: .shared_template_build_only
stage: build_first
.needs_samba-def-build:
extends: .shared_template_test_only
needs:
- job: samba-def-build
artifacts: true
- job: samba-codecheck
samba-mit-build:
extends: .shared_template_build_only
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora41}
stage: build_first
.needs_samba-mit-build:
extends: .shared_template_test_only
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora41}
needs:
- job: samba-mit-build
artifacts: true
- job: samba-codecheck
samba-h5l-build:
extends: .shared_template_build_only
.needs_samba-h5l-build:
extends: .shared_template_test_only
needs:
- job: samba-h5l-build
artifacts: true
samba-without-smb1-build:
extends: .shared_template_build_only
.needs_samba-without-smb1-build:
extends: .shared_template_test_only
needs:
- job: samba-without-smb1-build
artifacts: true
samba-nt4-build:
extends: .shared_template_build_only
.needs_samba-nt4-build:
extends: .shared_template_test_only
needs:
- job: samba-nt4-build
artifacts: true
samba-no-opath-build:
extends: .shared_template_build_only
.needs_samba-no-opath-build:
extends: .shared_template_test_only
needs:
- job: samba-no-opath-build
artifacts: true
samba:
extends: .shared_template
samba-mitkrb5:
extends: .shared_template
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora41}
samba-minimal-smbd:
extends: .shared_template
samba-nopython:
extends: .shared_template
samba-admem:
extends: .needs_samba-def-build
samba-ad-dc-2:
extends: .needs_samba-def-build
samba-ad-dc-3:
extends: .needs_samba-def-build
samba-ad-dc-4a:
extends: .needs_samba-def-build
samba-ad-dc-4b:
extends: .needs_samba-def-build
samba-ad-dc-5:
extends: .needs_samba-def-build
samba-ad-dc-6:
extends: .needs_samba-def-build
samba-ad-back1:
extends: .needs_samba-def-build
samba-ad-back2:
extends: .needs_samba-def-build
samba-schemaupgrade:
extends: .needs_samba-def-build
samba-libs:
extends: .shared_template
samba-fuzz:
extends: .shared_template
variables:
# We match what Google is running over at oss-fuzz
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2004}
ctdb:
extends: .shared_template
samba-ctdb:
extends: .shared_template
samba-ad-dc-ntvfs:
extends: .needs_samba-def-build
samba-admem-mit:
extends: .needs_samba-mit-build
samba-addc-mit-4a:
extends: .needs_samba-mit-build
samba-addc-mit-4b:
extends: .needs_samba-mit-build
# This task is run first to ensure we compile before we start the
# main run as it is the fastest full compile of Samba.
samba-fips:
extends: .shared_template
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora41}
samba-codecheck:
extends: .shared_template
needs:
stage: build_first
.private_test_only:
extends: .private_runner_test
stage: test_private
rules:
# See above, to avoid a duplicate CI on the MR (these rules override the others)
- if: $CI_MERGE_REQUEST_ID
when: never
# These jobs are only run if the gitlab repo has private runners available.
# To enable private jobs, you must add the following var and value to
# your gitlab repo by navigating to:
# settings -> CI/CD -> Environment variables
- if: $SUPPORT_PRIVATE_TEST == "yes"
.needs_ext4_support:
# All runners provide an ext4 filesystem
#
# Note: we don't use
# extends: .shared_template_test_only
# as that somehow resets the needs section
# and generates problems for something
# like this (which is used below)
#
# .needs_samba-SOME-build-ext4:
# extends:
# - .needs_samba-SOME-build
# - .needs_ext4_support
#
# So we only set stage again instead...
stage: test_only
.needs_5_15_kernel:
# Our private runners are based on
# ubuntu2204 with a 5.15 kernel.
#
# And they also provide an ext4 filesystem
extends: .private_test_only
.needs_samba-def-build-ext4:
extends:
- .needs_samba-def-build
- .needs_ext4_support
.needs_samba-mit-build-ext4:
extends:
- .needs_samba-mit-build
- .needs_ext4_support
.needs_samba-h5l-build-ext4:
extends:
- .needs_samba-h5l-build
- .needs_ext4_support
.needs_samba-without-smb1-build-5_15:
# Currently this doesn't strictly
# require a kernel >= 5.15, but only
# ext4 support.
#
# But we want to make sure that
# our private runners keep working
# and at least do a single job.
#
# In future we'll be able to run
# tests with io_uring in this
# setup, which will requires a
# 5.15 kernel in order to be useful.
extends:
- .needs_samba-without-smb1-build
- .needs_5_15_kernel
.needs_samba-nt4-build-ext4:
extends:
- .needs_samba-nt4-build
- .needs_ext4_support
.needs_samba-no-opath-build-ext4:
extends:
- .needs_samba-no-opath-build
- .needs_ext4_support
samba-fileserver:
extends: .needs_samba-h5l-build-ext4
samba-fileserver-without-smb1:
extends: .needs_samba-without-smb1-build-5_15
# This is a full build without the AD DC so we test the build with MIT
# Kerberos from the default system (Ubuntu 22.04 at this stage).
# Runtime behaviour checked via the ktest (static ccache and keytab)
# environment
samba-ktest-mit:
extends: .shared_template
samba-ad-dc-1:
extends: .needs_samba-def-build-ext4
samba-nt4:
extends: .needs_samba-nt4-build-ext4
samba-addc-mit-1:
extends: .needs_samba-mit-build-ext4
samba-no-opath1:
extends: .needs_samba-no-opath-build-ext4
samba-no-opath2:
extends: .needs_samba-no-opath-build-ext4
# 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages
pages:
extends: .shared_runner_build_image
stage: report
dependencies: # tell gitlab to download artifacts for these jobs
- others
- samba
- samba-mitkrb5
- samba-admem
- samba-ad-dc-2
- samba-ad-dc-3
- samba-ad-dc-4a
- samba-ad-dc-4b
- samba-ad-dc-5
- samba-ad-dc-6
- samba-libs
- samba-minimal-smbd
- samba-nopython
- samba-fuzz
# - ctdb # TODO
- samba-ctdb
- samba-ad-dc-ntvfs
- samba-admem-mit
- samba-addc-mit-4a
- samba-addc-mit-4b
- samba-ad-back1
- samba-ad-back2
- samba-fileserver
- samba-fileserver-without-smb1
- samba-ad-dc-1
- samba-nt4
- samba-schemaupgrade
- samba-addc-mit-1
- samba-fips
- samba-no-opath1
- samba-no-opath2
- ubuntu2204-samba-o3
script:
- ls -la *.info
- ./configure.developer
- make -j
- ls -la *.info
- lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info
- ls -la *.info
- genhtml all.info --ignore-errors source --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
artifacts:
expire_in: 30 days
paths:
- public
only:
variables:
- $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage"
# Coverity Scan
coverity:
extends: .shared_runner_build_image
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse155}
stage: build
script:
- wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
- tar xf /tmp/coverity_tool.tgz
- ./configure.developer --with-cluster-support
- cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
- tar czf cov-int.tar.gz cov-int
- curl
--form token=$COVERITY_SCAN_TOKEN
--form email=$COVERITY_SCAN_EMAIL
--form file=@cov-int.tar.gz
--form version="`git describe --tags`"
--form description="CI build"
https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
only:
refs:
- master
- schedules
variables:
- $COVERITY_SCAN_TOKEN != null
- $COVERITY_SCAN_PROJECT_NAME != null
- $COVERITY_SCAN_EMAIL != null
artifacts:
expire_in: 1 week
when: on_failure
paths:
- cov-int/*.txt
debian11-samba-32bit:
extends: .shared_template
variables:
AUTOBUILD_JOB_NAME: samba-32bit
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian11_32bit}
#
# We build samba-o3 on all supported distributions
#
# This job, which matches the main CI, needs to still do coverage so
# we show the coverage on the "none" environment tests
#
# We want --enable-coverage specified here otherwise we will have a
# different set of build options on the coverage build and can fail
# when -O3 gets combined with --enable-coverage in the scheduled
# builds.
ubuntu2204-samba-o3:
extends: .shared_template
variables:
AUTOBUILD_JOB_NAME: samba-o3
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2204}
SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: "--enable-coverage"
rules:
# See above, to avoid a duplicate CI on the MR (these rules override the others)
- if: $CI_MERGE_REQUEST_ID
when: never
# do not run o3 builds (which run a lot of VMs) if told not to
# (this uses the same variable as autobuild.py)
- if: $AUTOBUILD_SKIP_SAMBA_O3 == "1"
when: never
- when: on_success
# All other jobs do not want code coverage.
.samba-o3-template:
extends: .shared_template
variables:
AUTOBUILD_JOB_NAME: samba-o3
rules:
# See above, to avoid a duplicate CI on the MR (these rules override the others)
- if: $CI_MERGE_REQUEST_ID
when: never
# do not run o3 builds (which run a lot of VMs) if told not to
# (this uses the same variable as autobuild.py)
- if: $AUTOBUILD_SKIP_SAMBA_O3 == "1"
when: never
# do not run o3 for coverage since they are using different images
- if: $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
ubuntu2004-samba-o3:
extends: .samba-o3-template
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2004}
debian11-samba-o3:
extends: .samba-o3-template
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian11}
debian12-samba-o3:
extends: .samba-o3-template
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian12}
opensuse155-samba-o3:
extends: .samba-o3-template
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse155}
rocky8-samba-o3:
extends: .samba-o3-template
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_rocky8}
centos9s-samba-o3:
extends: .samba-o3-template
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos9s}
fedora41-samba-o3:
extends: .samba-o3-template
variables:
SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora41}
#
# Keep the samba-o3 sections at the end ...
#