2018-12-10 10:28:31 +03:00
#!/usr/bin/env python3
2010-09-25 23:51:51 +04:00
# run tests on all Samba subprojects and push to a git tree on success
# Copyright Andrew Tridgell 2010
# released under GNU GPL v3 or later
2018-03-09 17:06:21 +03:00
from __future__ import print_function
2019-05-22 03:15:40 +03:00
from subprocess import call , check_call , check_output , Popen , PIPE
2018-07-30 09:21:38 +03:00
import os
import tarfile
import sys
import time
2019-05-22 13:34:57 +03:00
import random
2010-09-25 23:51:51 +04:00
from optparse import OptionParser
2010-09-30 21:41:36 +04:00
import smtplib
2015-04-14 08:18:02 +03:00
import email
2010-09-30 21:41:36 +04:00
from email . mime . text import MIMEText
2015-04-14 08:18:02 +03:00
from email . mime . base import MIMEBase
from email . mime . application import MIMEApplication
from email . mime . multipart import MIMEMultipart
2012-04-09 08:52:03 +04:00
from distutils . sysconfig import get_python_lib
2015-04-14 08:18:02 +03:00
import platform
2010-09-25 23:51:51 +04:00
2018-09-03 13:07:23 +03:00
try :
from waflib . Build import CACHE_SUFFIX
except ImportError :
sys . path . insert ( 0 , " ./third_party/waf " )
from waflib . Build import CACHE_SUFFIX
2017-01-11 16:48:45 +03:00
os . environ [ " PYTHONUNBUFFERED " ] = " 1 "
2012-06-22 04:14:36 +04:00
# This speeds up testing remarkably.
os . environ [ ' TDB_NO_FSYNC ' ] = ' 1 '
2019-05-03 13:34:32 +03:00
def find_git_root ( ) :
''' get to the top of the git repo '''
p = os . getcwd ( )
while p != ' / ' :
if os . path . isdir ( os . path . join ( p , " .git " ) ) :
return p
p = os . path . abspath ( os . path . join ( p , ' .. ' ) )
return None
gitroot = find_git_root ( )
if gitroot is None :
raise Exception ( " Failed to find git root " )
2019-05-03 13:41:51 +03:00
def_testbase = os . getenv ( " AUTOBUILD_TESTBASE " , " /memdisk/ %s " % os . getenv ( ' USER ' ) )
parser = OptionParser ( )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --tail " , help = " show output while running " , default = False , action = " store_true " )
parser . add_option ( " --keeplogs " , help = " keep logs " , default = False , action = " store_true " )
parser . add_option ( " --nocleanup " , help = " don ' t remove test tree " , default = False , action = " store_true " )
parser . add_option ( " --testbase " , help = " base directory to run tests in (default %s ) " % def_testbase ,
2019-05-03 13:41:51 +03:00
default = def_testbase )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --passcmd " , help = " command to run on success " , default = None )
parser . add_option ( " --verbose " , help = " show all commands as they are run " ,
2019-05-03 13:41:51 +03:00
default = False , action = " store_true " )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --rebase " , help = " rebase on the given tree before testing " ,
2019-05-03 13:41:51 +03:00
default = None , type = ' str ' )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --pushto " , help = " push to a git url on success " ,
2019-05-03 13:41:51 +03:00
default = None , type = ' str ' )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --mark " , help = " add a Tested-By signoff before pushing " ,
2019-05-03 13:41:51 +03:00
default = False , action = " store_true " )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --fix-whitespace " , help = " fix whitespace on rebase " ,
2019-05-03 13:41:51 +03:00
default = False , action = " store_true " )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --retry " , help = " automatically retry if master changes " ,
2019-05-03 13:41:51 +03:00
default = False , action = " store_true " )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --email " , help = " send email to the given address on failure " ,
2019-05-03 13:41:51 +03:00
type = ' str ' , default = None )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --email-from " , help = " send email from the given address " ,
2019-05-03 13:41:51 +03:00
type = ' str ' , default = " autobuild@samba.org " )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --email-server " , help = " send email via the given server " ,
2019-05-03 13:41:51 +03:00
type = ' str ' , default = ' localhost ' )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --always-email " , help = " always send email, even on success " ,
2019-05-03 13:41:51 +03:00
action = " store_true " )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --daemon " , help = " daemonize after initial setup " ,
2019-05-03 13:41:51 +03:00
action = " store_true " )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --branch " , help = " the branch to work on (default=master) " ,
2019-05-03 13:41:51 +03:00
default = " master " , type = ' str ' )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --log-base " , help = " location where the logs can be found (default=cwd) " ,
2019-05-03 13:41:51 +03:00
default = gitroot , type = ' str ' )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --attach-logs " , help = " Attach logs to mails sent on success/failure? " ,
2019-05-03 13:41:51 +03:00
default = False , action = " store_true " )
2019-05-23 05:04:18 +03:00
parser . add_option ( " --restrict-tests " , help = " run as make test with this TESTS= regex " ,
2019-05-03 13:41:51 +03:00
default = ' ' )
2019-05-03 17:32:40 +03:00
parser . add_option ( " --enable-coverage " , dest = ' enable_coverage ' ,
action = " store_const " , const = ' --enable-coverage ' , default = ' ' ,
help = " Add --enable-coverage option while configure " )
2019-05-03 13:41:51 +03:00
( options , args ) = parser . parse_args ( )
if options . retry :
if options . rebase is None :
raise Exception ( ' You can only use --retry if you also rebase ' )
testbase = " %s /b %u " % ( options . testbase , os . getpid ( ) )
test_master = " %s /master " % testbase
test_prefix = " %s /prefix " % testbase
test_tmpdir = " %s /tmp " % testbase
os . environ [ ' TMPDIR ' ] = test_tmpdir
2019-05-03 18:00:53 +03:00
if options . enable_coverage :
LCOV_CMD = " cd $ {TEST_SOURCE_DIR} && lcov --capture --directory . --output-file $ {LOG_BASE} /$ {NAME} .info --rc ' geninfo_adjust_src_path=$ {TEST_SOURCE_DIR} / ' "
else :
LCOV_CMD = ' echo " lcov skipped since no --enable-coverage specified " '
2019-05-03 13:41:51 +03:00
2019-05-22 13:34:57 +03:00
if args :
# If we are only running specific test,
# do not sleep randomly to wait for it to start
def random_sleep ( low , high ) :
return ' sleep 1 '
else :
def random_sleep ( low , high ) :
return ' sleep {} ' . format ( random . randint ( low , high ) )
2010-09-25 23:51:51 +04:00
cleanup_list = [ ]
2011-02-07 02:05:32 +03:00
builddirs = {
2018-07-30 09:17:14 +03:00
" ctdb " : " ctdb " ,
" ldb " : " lib/ldb " ,
" tdb " : " lib/tdb " ,
" talloc " : " lib/talloc " ,
" replace " : " lib/replace " ,
" tevent " : " lib/tevent " ,
2018-08-16 04:54:14 +03:00
" pidl " : " pidl "
2018-07-30 09:14:37 +03:00
}
2011-02-07 02:05:32 +03:00
2014-11-11 16:55:55 +03:00
ctdb_configure_params = " --enable-developer $ {PREFIX} "
samba_configure_params = " $ {ENABLE_COVERAGE} $ {PREFIX} --with-profiling-data "
2011-02-07 02:05:32 +03:00
2018-11-24 14:13:47 +03:00
samba_libs_envvars = " PYTHONPATH=$ {PYTHON_PREFIX} :$PYTHONPATH "
2015-08-13 18:34:42 +03:00
samba_libs_envvars + = " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ {PREFIX_DIR} /lib/pkgconfig "
2015-08-14 09:40:37 +03:00
samba_libs_envvars + = " ADDITIONAL_CFLAGS= ' -Wmissing-prototypes ' "
2014-11-11 16:55:55 +03:00
samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check $ {ENABLE_COVERAGE} --enable-debug -C $ {PREFIX} "
2019-02-15 01:58:51 +03:00
samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE "
2018-07-27 06:50:15 +03:00
samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt "
2019-02-15 01:58:51 +03:00
samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
2016-02-01 14:55:34 +03:00
2019-05-22 14:29:56 +03:00
def format_option ( name , value = None ) :
""" Format option as str list. """
if value is None : # boolean option
return [ name ]
if not isinstance ( value , list ) : # single value option
value = [ value ]
# repeatable option
return [ ' {} = {} ' . format ( name , item ) for item in value ]
def make_test (
cmd = ' make test ' ,
FAIL_IMMEDIATELY = 1 ,
TESTS = ' ' ,
include_envs = None ,
exclude_envs = None ) :
test_options = [ ]
if include_envs :
test_options = format_option ( ' --include-env ' , include_envs )
if exclude_envs :
test_options = format_option ( ' --exclude-env ' , exclude_envs )
if test_options :
# join envs options to original test options
TESTS = ( TESTS + ' ' + ' ' . join ( test_options ) ) . strip ( )
_options = [ ]
if FAIL_IMMEDIATELY :
_options . append ( ' FAIL_IMMEDIATELY=1 ' )
if TESTS :
_options . append ( " TESTS= ' {} ' " . format ( TESTS ) )
return ' ' . join ( [ cmd ] + _options )
2019-12-10 00:59:49 +03:00
# When updating this list, also update .gitlab-ci.yml to add the job
# and to make it a dependency of 'page' for the coverage report.
2010-09-25 23:51:51 +04:00
tasks = {
2019-05-22 13:11:28 +03:00
" ctdb " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure " + ctdb_configure_params ) ,
( " make " , " make all " ) ,
( " install " , " make install " ) ,
( " test " , " make autotest " ) ,
( " check-clean-tree " , " ../script/clean-source-tree.sh " ) ,
( " clean " , " make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
2011-10-13 00:27:55 +04:00
2019-01-23 11:43:33 +03:00
# We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
" samba " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( exclude_envs = [
" none " ,
" nt4_dc " ,
" nt4_dc_schannel " ,
" nt4_member " ,
" ad_dc " ,
" ad_dc_backup " ,
" ad_dc_ntvfs " ,
" ad_dc_default " ,
" ad_dc_slowtests " ,
" ad_dc_no_nss " ,
" ad_dc_no_ntlm " ,
" fl2003dc " ,
" fl2008dc " ,
" fl2008r2dc " ,
" ad_member " ,
" ad_member_idmap_rid " ,
" ad_member_idmap_ad " ,
2019-11-04 09:47:15 +03:00
" ad_member_rfc2307 " ,
" chgdcpass " ,
" vampire_2000_dc " ,
" fl2000dc " ,
" fileserver " ,
" maptoguest " ,
" simpleserver " ,
" backupfromdc " ,
" restoredc " ,
" renamedc " ,
" offlinebackupdc " ,
" labdc " ,
" preforkrestartdc " ,
" proclimitdc " ,
" promoted_dc " ,
" vampire_dc " ,
" rodc " ,
" ad_dc_default " ,
" ad_dc_slowtests " ,
" schema_pair_dc " ,
" schema_dc " ,
2020-01-28 12:18:37 +03:00
" clusteredmember " ,
2019-11-04 09:47:15 +03:00
] ) ) ,
( " lcov " , LCOV_CMD ) ,
( " install " , " make install " ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
( " clean " , " make clean " ) ,
] ,
# We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
" samba-mitkrb5 " : [
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc " + samba_configure_params ) ,
( " make " , " make -j " ) ,
( " test " , make_test ( exclude_envs = [
" none " ,
" nt4_dc " ,
" nt4_dc_schannel " ,
" nt4_member " ,
" ad_dc " ,
" ad_dc_backup " ,
" ad_dc_ntvfs " ,
" ad_dc_default " ,
" ad_dc_slowtests " ,
" ad_dc_no_nss " ,
" ad_dc_no_ntlm " ,
" fl2003dc " ,
" fl2008dc " ,
" fl2008r2dc " ,
" ad_member " ,
" ad_member_idmap_rid " ,
" ad_member_idmap_ad " ,
2019-05-22 14:29:56 +03:00
" ad_member_rfc2307 " ,
" chgdcpass " ,
" vampire_2000_dc " ,
" fl2000dc " ,
" fileserver " ,
" maptoguest " ,
" simpleserver " ,
" backupfromdc " ,
" restoredc " ,
" renamedc " ,
" offlinebackupdc " ,
" labdc " ,
" preforkrestartdc " ,
" proclimitdc " ,
" promoted_dc " ,
" vampire_dc " ,
" rodc " ,
" ad_dc_default " ,
" ad_dc_slowtests " ,
" schema_pair_dc " ,
" schema_dc " ,
2020-01-28 12:18:37 +03:00
" clusteredmember " ,
2019-05-22 14:29:56 +03:00
] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " install " , " make install " ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
( " clean " , " make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
" samba-nt4 " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
2020-03-21 22:15:30 +03:00
( " configure " , " ./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
2019-05-22 13:17:28 +03:00
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [
" nt4_dc " ,
" nt4_dc_schannel " ,
" nt4_member " ,
2020-03-21 22:15:30 +03:00
" simpleserver " ,
2019-05-22 14:29:56 +03:00
] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " install " , " make install " ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
( " clean " , " make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
2019-07-30 13:00:18 +03:00
" samba-fileserver " : [
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
2020-03-12 06:07:01 +03:00
( " configure " , " ./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
2019-07-30 13:00:18 +03:00
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [
" fileserver " ,
" maptoguest " ,
2020-03-12 06:07:01 +03:00
" ktest " , # ktest is also tested in samba and samba-mitkrb5
# but is tested here against a system Heimdal
2019-10-30 23:45:55 +03:00
] ) ) ,
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
] ,
2019-10-22 12:49:22 +03:00
" samba-admem " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [
" ad_member " ,
" ad_member_idmap_rid " ,
" ad_member_idmap_ad " ,
" ad_member_rfc2307 " ,
] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
2019-05-22 13:11:28 +03:00
] ,
" samba-ad-dc-1 " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 1 , 1 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [
" ad_dc " ,
" ad_dc_no_nss " ,
" ad_dc_no_ntlm " ,
] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
2019-05-22 13:11:28 +03:00
] ,
" samba-ad-dc-2 " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 1 , 1 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [
" vampire_dc " ,
" vampire_2000_dc " ,
" rodc " ,
] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
2019-05-22 13:11:28 +03:00
] ,
" samba-ad-dc-3 " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 1 , 1 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [
" promoted_dc " ,
" chgdcpass " ,
" preforkrestartdc " ,
" proclimitdc " ,
] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
2019-05-22 13:11:28 +03:00
] ,
" samba-ad-dc-4 " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 1 , 1 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [
" fl2000dc " ,
" fl2003dc " ,
" fl2008dc " ,
" fl2008r2dc " ,
] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
2019-05-22 13:11:28 +03:00
] ,
" samba-ad-dc-5 " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 1 , 1 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [ " ad_dc_default " ] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
2019-05-22 13:11:28 +03:00
] ,
" samba-ad-dc-6 " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 1 , 1 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [ " ad_dc_slowtests " ] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
2019-05-22 13:11:28 +03:00
] ,
" samba-schemaupgrade " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 1 , 1 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [ " schema_dc " , " schema_pair_dc " ] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
2019-05-22 13:11:28 +03:00
] ,
2019-04-02 00:45:36 +03:00
2019-01-23 11:43:33 +03:00
# We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
# This is currently the longest task, so we don't randomly delay it.
" samba-ad-dc-ntvfs " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 1 , 1 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [ " ad_dc_ntvfs " ] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
2019-05-22 13:11:28 +03:00
] ,
2019-01-23 11:43:33 +03:00
2019-02-05 05:17:03 +03:00
# run the backup/restore testenvs separately as they're fairly standalone
# (and CI seems to max out at ~8 different DCs running at once)
2019-05-22 13:11:28 +03:00
" samba-ad-dc-backup " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( include_envs = [
" backupfromdc " ,
" restoredc " ,
" renamedc " ,
" offlinebackupdc " ,
" labdc " ,
" ad_dc_backup " ,
] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
2019-05-22 13:11:28 +03:00
] ,
2019-10-22 12:49:22 +03:00
" samba-admem-mit " : [
2019-10-09 16:55:50 +03:00
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc " + samba_configure_params ) ,
( " make " , " make -j " ) ,
( " test " , make_test ( include_envs = [
" ad_member " ,
" ad_member_idmap_rid " ,
" ad_member_idmap_ad " ,
" ad_member_rfc2307 " ,
] ) ) ,
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
] ,
" samba-ad-dc-1-mitkrb5 " : [
( " random-sleep " , random_sleep ( 1 , 1 ) ) ,
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc " + samba_configure_params ) ,
( " make " , " make -j " ) ,
( " test " , make_test ( include_envs = [
" ad_dc " ,
" ad_dc_no_nss " ,
" ad_dc_no_ntlm " ,
] ) ) ,
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
] ,
" samba-ad-dc-4-mitkrb5 " : [
( " random-sleep " , random_sleep ( 1 , 1 ) ) ,
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc " + samba_configure_params ) ,
( " make " , " make -j " ) ,
( " test " , make_test ( include_envs = [
" fl2000dc " ,
" fl2003dc " ,
" fl2008dc " ,
" fl2008r2dc " ,
] ) ) ,
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
] ,
2019-05-22 13:11:28 +03:00
" samba-test-only " : [
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " test " , make_test ( TESTS = " $ {TESTS} " ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
2019-05-22 13:11:28 +03:00
] ,
2016-07-27 05:28:04 +03:00
2015-05-04 10:21:53 +03:00
# Test cross-compile infrastructure
2019-05-22 13:11:28 +03:00
" samba-xc " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 900 , 1500 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure-native " , " ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params ) ,
2019-05-22 13:11:28 +03:00
( " configure-cross-execute " , " ./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh " \
2019-05-22 13:17:28 +03:00
" --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab " + samba_configure_params ) ,
2019-10-09 21:53:43 +03:00
( " verify-cross-execute-output " , " grep ' ^Checking value of NSIG ' ./bin-xe/cross-answers.txt " ) ,
2019-05-22 13:11:28 +03:00
( " configure-cross-answers " , " ./configure.developer --out ./bin-xa --cross-compile " \
2019-05-22 13:17:28 +03:00
" --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab " + samba_configure_params ) ,
2019-05-22 13:11:28 +03:00
( " compare-results " , " script/compare_cc_results.py "
" ./bin/c4che/default {} "
" ./bin-xe/c4che/default {} "
2019-05-22 13:17:28 +03:00
" ./bin-xa/c4che/default {} " . format ( * ( [ CACHE_SUFFIX ] * 3 ) ) ) ,
2019-10-09 21:53:43 +03:00
( " modify-cross-answers " , " sed -i.bak -e ' s/^ \\ (Checking value of NSIG: \\ ) .*/ \\ 1 \" 1234 \" / ' ./bin-xe/cross-answers.txt " ) ,
( " configure-cross-answers-modified " , " ./configure.developer --out ./bin-xa2 --cross-compile " \
" --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab " + samba_configure_params ) ,
( " verify-cross-answers " , " test $(sed -n -e ' s/VALUEOF_NSIG = \\ (.* \\ )/ \\ 1/p ' ./bin-xa2/c4che/default {} ) " \
" = \" ' 1234 ' \" " . format ( CACHE_SUFFIX ) ) ,
( " invalidate-cross-answers " , " sed -i.bak -e ' /^Checking value of NSIG/d ' ./bin-xe/cross-answers.txt " ) ,
( " configure-cross-answers-fail " , " ./configure.developer --out ./bin-xa3 --cross-compile " \
" --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab " + samba_configure_params + \
" ; test $? -ne 0 " ) ,
2019-05-22 13:11:28 +03:00
] ,
2015-05-04 10:21:53 +03:00
2017-06-30 02:11:05 +03:00
# test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
2019-05-22 13:11:28 +03:00
" samba-o3 " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ADDITIONAL_CFLAGS= ' -O3 -Wp,-D_FORTIFY_SOURCE=2 ' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable " + samba_configure_params ) ,
( " make " , " make -j " ) ,
2019-11-22 17:23:35 +03:00
( " test " , make_test ( cmd = ' make test ' , include_envs = [ " none " ] ) ) ,
( " quicktest " , make_test ( cmd = ' make quicktest ' , include_envs = [ " ad_dc " ] ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
( " install " , " make install " ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
( " clean " , " make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
" samba-ctdb " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 900 , 1500 ) ) ,
2019-05-22 13:11:28 +03:00
# make sure we have tdb around:
2019-05-22 13:17:28 +03:00
( " tdb-configure " , " cd lib/tdb && PYTHONPATH=$ {PYTHON_PREFIX} :$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ {PREFIX_DIR} /lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C $ {PREFIX} " ) ,
( " tdb-make " , " cd lib/tdb && make " ) ,
( " tdb-install " , " cd lib/tdb && make install " ) ,
2019-05-22 13:11:28 +03:00
# build samba with cluster support (also building ctdb):
2020-02-14 19:19:46 +03:00
( " samba-configure " ,
" PYTHONPATH=$ {PYTHON_PREFIX} :$PYTHONPATH "
" PKG_CONFIG_PATH=$ {PREFIX_DIR} /lib/pkgconfig:$ {PKG_CONFIG_PATH} "
" ./configure.developer $ {PREFIX} "
" --with-selftest-prefix=./bin/ab "
" --with-cluster-support "
2020-02-14 19:20:22 +03:00
" --without-ad-dc "
2020-02-14 19:19:46 +03:00
" --bundled-libraries=!tdb " ) ,
2019-05-22 13:17:28 +03:00
( " samba-make " , " make " ) ,
( " samba-check " , " ./bin/smbd -b | grep CLUSTER_SUPPORT " ) ,
( " samba-install " , " make install " ) ,
( " ctdb-check " , " test -e $ {PREFIX_DIR} /sbin/ctdbd " ) ,
2019-05-22 13:11:28 +03:00
2018-10-16 14:56:14 +03:00
( " test " ,
make_test ( cmd = ' make test ' ,
include_envs = [ " clusteredmember " ] )
) ,
2019-05-22 13:11:28 +03:00
# clean up:
2019-05-22 13:17:28 +03:00
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
( " clean " , " make clean " ) ,
( " ctdb-clean " , " cd ./ctdb && make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
2012-06-20 16:05:50 +04:00
2018-08-22 00:35:58 +03:00
" samba-libs " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
2019-05-22 13:17:28 +03:00
( " talloc-configure " , " cd lib/talloc && " + samba_libs_configure_libs ) ,
( " talloc-make " , " cd lib/talloc && make " ) ,
( " talloc-install " , " cd lib/talloc && make install " ) ,
2019-05-22 13:11:28 +03:00
2019-05-22 13:17:28 +03:00
( " tdb-configure " , " cd lib/tdb && " + samba_libs_configure_libs ) ,
( " tdb-make " , " cd lib/tdb && make " ) ,
( " tdb-install " , " cd lib/tdb && make install " ) ,
2019-05-22 13:11:28 +03:00
2019-05-22 13:17:28 +03:00
( " tevent-configure " , " cd lib/tevent && " + samba_libs_configure_libs ) ,
( " tevent-make " , " cd lib/tevent && make " ) ,
( " tevent-install " , " cd lib/tevent && make install " ) ,
2019-05-22 13:11:28 +03:00
2019-05-22 13:17:28 +03:00
( " ldb-configure " , " cd lib/ldb && " + samba_libs_configure_libs ) ,
( " ldb-make " , " cd lib/ldb && make " ) ,
( " ldb-install " , " cd lib/ldb && make install " ) ,
2019-05-22 13:11:28 +03:00
2019-05-22 13:17:28 +03:00
( " nondevel-configure " , " ./configure $ {PREFIX} " ) ,
( " nondevel-make " , " make -j " ) ,
( " nondevel-check " , " ./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0 " ) ,
( " nondevel-install " , " make install " ) ,
( " nondevel-dist " , " make dist " ) ,
2019-05-22 13:11:28 +03:00
# retry with all modules shared
2019-05-22 13:17:28 +03:00
( " allshared-distclean " , " make distclean " ) ,
( " allshared-configure " , samba_libs_configure_samba + " --with-shared-modules=ALL " ) ,
( " allshared-make " , " make -j " ) ,
2019-05-22 13:11:28 +03:00
] ,
2015-08-13 18:38:43 +03:00
2018-08-22 00:35:58 +03:00
" samba-static " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 1 , 1 ) ) ,
2019-05-22 13:11:28 +03:00
# build with all modules static
2019-05-22 13:17:28 +03:00
( " allstatic-configure " , " ./configure.developer " + samba_configure_params + " --with-static-modules=ALL " ) ,
( " allstatic-make " , " make -j " ) ,
2019-05-22 14:29:56 +03:00
( " allstatic-test " , make_test ( TESTS = " samba3.smb2.create.*nt4_dc " ) ) ,
2019-05-22 13:17:28 +03:00
( " lcov " , LCOV_CMD ) ,
2019-05-22 13:11:28 +03:00
# retry without any required modules
2019-05-22 13:17:28 +03:00
( " none-distclean " , " make distclean " ) ,
( " none-configure " , " ./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT " ) ,
( " none-make " , " make -j " ) ,
2019-05-22 13:11:28 +03:00
# retry with nonshared smbd and smbtorture
2019-05-22 13:17:28 +03:00
( " nonshared-distclean " , " make distclean " ) ,
2020-02-02 23:51:12 +03:00
( " nonshared-configure " , " ./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd " ) ,
2019-12-04 12:52:06 +03:00
( " nonshared-make " , " make -j " )
] ,
" samba-fuzz " : [
# build the fuzzers (static) via the oss-fuzz script
( " fuzzers-mkdir-prefix " , " mkdir -p $ {PREFIX_DIR} " ) ,
( " fuzzers-build " , " OUT=$ {PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl " ) ,
( " fuzzers-check " , " ./lib/fuzzing/oss-fuzz/check_build.sh $ {PREFIX_DIR} " )
2019-05-22 13:11:28 +03:00
] ,
2012-04-09 08:52:03 +04:00
2017-01-30 17:36:31 +03:00
# Test Samba without python still builds. When this test fails
# due to more use of Python, the expectations is that the newly
# failing part of the code should be disabled when
# --disable-python is set (rather than major work being done to
# support this environment). The target here is for vendors
# shipping a minimal smbd.
2018-08-22 00:35:58 +03:00
" samba-nopython " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
2014-11-11 16:55:55 +03:00
( " configure " , " ./configure.developer $ {ENABLE_COVERAGE} $ {PREFIX} --with-profiling-data --disable-python --without-ad-dc " ) ,
2019-05-22 13:17:28 +03:00
( " make " , " make -j " ) ,
( " install " , " make install " ) ,
( " find-python " , " script/find_python.sh $ {PREFIX} " ) ,
( " test " , " make test-nopython " ) ,
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
( " clean " , " make clean " ) ,
( " talloc-configure " , " cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python " ) ,
( " talloc-make " , " cd lib/talloc && make " ) ,
( " talloc-install " , " cd lib/talloc && make install " ) ,
( " tdb-configure " , " cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python " ) ,
( " tdb-make " , " cd lib/tdb && make " ) ,
( " tdb-install " , " cd lib/tdb && make install " ) ,
( " tevent-configure " , " cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python " ) ,
( " tevent-make " , " cd lib/tevent && make " ) ,
( " tevent-install " , " cd lib/tevent && make install " ) ,
( " ldb-configure " , " cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python " ) ,
( " ldb-make " , " cd lib/ldb && make " ) ,
( " ldb-install " , " cd lib/ldb && make install " ) ,
2019-05-22 13:11:28 +03:00
# retry against installed library packages
2019-05-22 13:17:28 +03:00
( " libs-configure " , samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc " ) ,
( " libs-make " , " make -j " ) ,
( " libs-install " , " make install " ) ,
( " libs-check-clean-tree " , " script/clean-source-tree.sh " ) ,
( " libs-clean " , " make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
2017-01-30 17:36:31 +03:00
2019-02-15 02:17:49 +03:00
# check we can do the same thing using python2
" samba-nopython-py2 " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 300 , 900 ) ) ,
2014-11-11 16:55:55 +03:00
( " configure " , " PYTHON=python2 ./configure.developer $ {ENABLE_COVERAGE} $ {PREFIX} --with-profiling-data --disable-python --without-ad-dc " ) ,
2019-05-22 13:17:28 +03:00
( " make " , " PYTHON=python2 make -j " ) ,
( " install " , " PYTHON=python2 make install " ) ,
( " find-python " , " script/find_python.sh $ {PREFIX} " ) ,
( " test " , " make test-nopython " ) ,
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " script/clean-source-tree.sh " ) ,
( " clean " , " PYTHON=python2 make clean " ) ,
( " talloc-configure " , " cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python " ) ,
( " talloc-make " , " cd lib/talloc && PYTHON=python2 make " ) ,
( " talloc-install " , " cd lib/talloc && PYTHON=python2 make install " ) ,
( " tdb-configure " , " cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python " ) ,
( " tdb-make " , " cd lib/tdb && PYTHON=python2 make " ) ,
( " tdb-install " , " cd lib/tdb && PYTHON=python2 make install " ) ,
( " tevent-configure " , " cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python " ) ,
( " tevent-make " , " cd lib/tevent && PYTHON=python2 make " ) ,
( " tevent-install " , " cd lib/tevent && PYTHON=python2 make install " ) ,
( " ldb-configure " , " cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python " ) ,
( " ldb-make " , " cd lib/ldb && PYTHON=python2 make " ) ,
( " ldb-install " , " cd lib/ldb && PYTHON=python2 make install " ) ,
2019-05-22 13:11:28 +03:00
# retry against installed library packages
2019-05-22 13:17:28 +03:00
( " libs-configure " , " PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc " ) ,
( " libs-make " , " PYTHON=python2 make -j " ) ,
( " libs-install " , " PYTHON=python2 make install " ) ,
( " libs-check-clean-tree " , " script/clean-source-tree.sh " ) ,
( " libs-clean " , " PYTHON=python2 make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
2016-09-26 03:46:56 +03:00
2018-08-22 00:35:58 +03:00
" ldb " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 60 , 600 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure $ {ENABLE_COVERAGE} --enable-developer -C $ {PREFIX} " ) ,
( " make " , " make " ) ,
( " install " , " make install " ) ,
( " test " , " make test " ) ,
( " lcov " , LCOV_CMD ) ,
( " clean " , " make clean " ) ,
( " configure-no-lmdb " , " ./configure $ {ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C $ {PREFIX} " ) ,
( " make-no-lmdb " , " make " ) ,
( " test-no-lmdb " , " make test " ) ,
( " lcov-no-lmdb " , LCOV_CMD ) ,
( " install-no-lmdb " , " make install " ) ,
( " check-clean-tree " , " ../../script/clean-source-tree.sh " ) ,
( " distcheck " , " make distcheck " ) ,
( " clean " , " make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
2010-10-01 14:19:56 +04:00
2018-08-22 00:35:58 +03:00
" tdb " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 60 , 600 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure $ {ENABLE_COVERAGE} --enable-developer -C $ {PREFIX} " ) ,
( " make " , " make " ) ,
( " install " , " make install " ) ,
( " test " , " make test " ) ,
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " ../../script/clean-source-tree.sh " ) ,
( " distcheck " , " make distcheck " ) ,
( " clean " , " make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
2012-03-07 06:32:22 +04:00
2018-08-22 00:35:58 +03:00
" talloc " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 60 , 600 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure $ {ENABLE_COVERAGE} --enable-developer -C $ {PREFIX} " ) ,
( " make " , " make " ) ,
( " install " , " make install " ) ,
( " test " , " make test " ) ,
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " ../../script/clean-source-tree.sh " ) ,
( " distcheck " , " make distcheck " ) ,
( " clean " , " make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
2011-02-07 02:05:32 +03:00
2018-08-22 00:35:58 +03:00
" replace " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 60 , 600 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure $ {ENABLE_COVERAGE} --enable-developer -C $ {PREFIX} " ) ,
( " make " , " make " ) ,
( " install " , " make install " ) ,
( " test " , " make test " ) ,
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " ../../script/clean-source-tree.sh " ) ,
( " distcheck " , " make distcheck " ) ,
( " clean " , " make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
2010-10-01 14:19:56 +04:00
2018-08-22 00:35:58 +03:00
" tevent " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 60 , 600 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " ./configure $ {ENABLE_COVERAGE} --enable-developer -C $ {PREFIX} " ) ,
( " make " , " make " ) ,
( " install " , " make install " ) ,
( " test " , " make test " ) ,
( " lcov " , LCOV_CMD ) ,
( " check-clean-tree " , " ../../script/clean-source-tree.sh " ) ,
( " distcheck " , " make distcheck " ) ,
( " clean " , " make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
2011-01-05 11:30:48 +03:00
2018-07-30 09:17:14 +03:00
" pidl " : [
2019-05-22 13:34:57 +03:00
( " random-sleep " , random_sleep ( 60 , 600 ) ) ,
2019-05-22 13:17:28 +03:00
( " configure " , " perl Makefile.PL PREFIX=$ {PREFIX_DIR} " ) ,
( " touch " , " touch *.yp " ) ,
( " make " , " make " ) ,
( " test " , " make test " ) ,
( " install " , " make install " ) ,
( " checkout-yapp-generated " , " git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm " ) ,
( " check-clean-tree " , " ../script/clean-source-tree.sh " ) ,
( " clean " , " make clean " ) ,
2019-05-22 13:11:28 +03:00
] ,
2018-10-16 21:25:26 +03:00
2011-02-07 02:05:32 +03:00
# these are useful for debugging autobuild
2019-05-22 13:17:28 +03:00
' pass ' : [ ( " pass " , ' echo passing && /bin/true ' ) ] ,
' fail ' : [ ( " fail " , ' echo failing && /bin/false ' ) ] ,
2010-09-25 23:51:51 +04:00
}
2019-12-10 00:59:49 +03:00
defaulttasks = list ( tasks . keys ( ) )
2019-12-04 12:52:06 +03:00
2019-12-10 00:59:49 +03:00
defaulttasks . remove ( " pass " )
defaulttasks . remove ( " fail " )
defaulttasks . remove ( " samba-test-only " )
2019-12-04 12:52:06 +03:00
defaulttasks . remove ( " samba-fuzz " )
2019-12-10 00:59:49 +03:00
if os . environ . get ( " AUTOBUILD_SKIP_SAMBA_O3 " , " 0 " ) == " 1 " :
defaulttasks . remove ( " samba-o3 " )
2018-07-30 09:20:39 +03:00
2017-01-11 17:02:17 +03:00
def do_print ( msg ) :
2018-03-09 17:06:21 +03:00
print ( " %s " % msg )
2017-01-11 17:02:17 +03:00
sys . stdout . flush ( )
sys . stderr . flush ( )
2018-07-30 09:20:39 +03:00
2010-09-26 23:50:06 +04:00
def run_cmd ( cmd , dir = " . " , show = None , output = False , checkfail = True ) :
2010-09-25 23:51:51 +04:00
if show is None :
show = options . verbose
if show :
2017-01-11 17:02:17 +03:00
do_print ( " Running: ' %s ' in ' %s ' " % ( cmd , dir ) )
2010-09-26 22:46:01 +04:00
if output :
2019-05-22 03:15:40 +03:00
out = check_output ( [ cmd ] , shell = True , cwd = dir )
return out . decode ( encoding = ' utf-8 ' , errors = ' backslashreplace ' )
2010-10-01 19:45:47 +04:00
elif checkfail :
2010-10-01 22:12:24 +04:00
return check_call ( cmd , shell = True , cwd = dir )
2010-10-01 19:45:47 +04:00
else :
2010-10-01 22:12:24 +04:00
return call ( cmd , shell = True , cwd = dir )
2010-10-01 19:45:47 +04:00
2010-09-25 23:51:51 +04:00
2010-10-01 14:19:56 +04:00
class builder ( object ) :
2010-09-25 23:51:51 +04:00
''' handle build of one directory '''
2010-10-01 14:19:56 +04:00
2019-02-15 02:20:10 +03:00
def __init__ ( self , name , sequence , cp = True ) :
2010-09-25 23:51:51 +04:00
self . name = name
2019-05-21 04:56:12 +03:00
self . dir = builddirs . get ( name , ' . ' )
2010-09-25 23:51:51 +04:00
self . tag = self . name . replace ( ' / ' , ' _ ' )
self . sequence = sequence
self . next = 0
2010-09-30 21:41:36 +04:00
self . stdout_path = " %s / %s .stdout " % ( gitroot , self . tag )
self . stderr_path = " %s / %s .stderr " % ( gitroot , self . tag )
if options . verbose :
2017-01-11 17:02:17 +03:00
do_print ( " stdout for %s in %s " % ( self . name , self . stdout_path ) )
do_print ( " stderr for %s in %s " % ( self . name , self . stderr_path ) )
2010-09-25 23:51:51 +04:00
run_cmd ( " rm -f %s %s " % ( self . stdout_path , self . stderr_path ) )
self . stdout = open ( self . stdout_path , ' w ' )
self . stderr = open ( self . stderr_path , ' w ' )
self . stdin = open ( " /dev/null " , ' r ' )
2019-05-03 08:05:08 +03:00
self . test_source_dir = " %s / %s " % ( testbase , self . tag )
2019-05-22 14:50:01 +03:00
self . cwd = " %s / %s " % ( self . test_source_dir , self . dir )
2016-12-16 15:35:01 +03:00
self . prefix = " %s / %s " % ( test_prefix , self . tag )
2019-05-03 08:05:08 +03:00
run_cmd ( " rm -rf %s " % self . test_source_dir )
2017-01-11 16:42:08 +03:00
run_cmd ( " rm -rf %s " % self . prefix )
2016-01-04 04:20:54 +03:00
if cp :
2019-05-03 08:05:08 +03:00
run_cmd ( " cp --recursive --link --archive %s %s " % ( test_master , self . test_source_dir ) , dir = test_master , show = True )
2016-01-04 04:20:54 +03:00
else :
2019-05-03 08:05:08 +03:00
run_cmd ( " git clone --recursive --shared %s %s " % ( test_master , self . test_source_dir ) , dir = test_master , show = True )
2010-09-25 23:51:51 +04:00
self . start_next ( )
def start_next ( self ) :
if self . next == len ( self . sequence ) :
2017-01-11 16:42:08 +03:00
if not options . nocleanup :
2019-05-03 08:05:08 +03:00
run_cmd ( " rm -rf %s " % self . test_source_dir )
2017-01-11 16:42:08 +03:00
run_cmd ( " rm -rf %s " % self . prefix )
2017-01-11 17:02:17 +03:00
do_print ( ' %s : Completed OK ' % self . name )
2010-09-25 23:51:51 +04:00
self . done = True
return
2019-05-22 13:17:28 +03:00
( self . stage , self . cmd ) = self . sequence [ self . next ]
2018-11-24 14:13:47 +03:00
self . cmd = self . cmd . replace ( " $ {PYTHON_PREFIX} " , get_python_lib ( plat_specific = 1 , standard_lib = 0 , prefix = self . prefix ) )
2010-10-01 14:19:56 +04:00
self . cmd = self . cmd . replace ( " $ {PREFIX} " , " --prefix= %s " % self . prefix )
2011-01-05 11:30:48 +03:00
self . cmd = self . cmd . replace ( " $ {PREFIX_DIR} " , " %s " % self . prefix )
2016-07-27 05:28:04 +03:00
self . cmd = self . cmd . replace ( " $ {TESTS} " , options . restrict_tests )
2019-05-06 06:14:43 +03:00
self . cmd = self . cmd . replace ( " $ {TEST_SOURCE_DIR} " , self . test_source_dir )
self . cmd = self . cmd . replace ( " $ {LOG_BASE} " , options . log_base )
self . cmd = self . cmd . replace ( " $ {NAME} " , self . name )
self . cmd = self . cmd . replace ( " $ {ENABLE_COVERAGE} " , options . enable_coverage )
2019-05-22 14:50:01 +03:00
do_print ( ' %s : [ %s ] Running %s in %r ' % ( self . name , self . stage , self . cmd , self . cwd ) )
2010-09-25 23:51:51 +04:00
self . proc = Popen ( self . cmd , shell = True ,
2019-05-22 14:50:01 +03:00
close_fds = True , cwd = self . cwd ,
2010-09-25 23:51:51 +04:00
stdout = self . stdout , stderr = self . stderr , stdin = self . stdin )
self . next + = 1
2010-10-01 14:19:56 +04:00
class buildlist ( object ) :
2010-09-25 23:51:51 +04:00
''' handle build of multiple directories '''
2010-10-01 14:19:56 +04:00
2016-07-27 05:09:08 +03:00
def __init__ ( self , tasknames , rebase_url , rebase_branch = " master " ) :
2010-09-25 23:51:51 +04:00
self . tail_proc = None
2010-09-26 04:35:55 +04:00
self . retry = None
2019-05-21 04:56:12 +03:00
if not tasknames :
2016-07-27 05:28:04 +03:00
if options . restrict_tests :
tasknames = [ " samba-test-only " ]
else :
tasknames = defaulttasks
2015-04-14 02:59:57 +03:00
2019-05-21 04:56:12 +03:00
self . tlist = [ builder ( n , tasks [ n ] , cp = ( n != " pidl " ) ) for n in tasknames ]
2010-09-26 04:35:55 +04:00
if options . retry :
2012-09-22 02:23:21 +04:00
rebase_remote = " rebaseon "
2018-07-30 09:16:43 +03:00
retry_task = [ ( " retry " ,
2012-09-22 02:23:21 +04:00
''' set -e
git remote add - t % s % s % s
git fetch % s
while : ; do
sleep 60
git describe % s / % s > old_remote_branch . desc
git fetch % s
git describe % s / % s > remote_branch . desc
diff old_remote_branch . desc remote_branch . desc
done
''' % (
2019-05-22 13:17:28 +03:00
rebase_branch , rebase_remote , rebase_url ,
2012-09-22 02:23:21 +04:00
rebase_remote ,
rebase_remote , rebase_branch ,
rebase_remote ,
rebase_remote , rebase_branch
2019-05-22 13:17:28 +03:00
) ) ]
2012-09-22 02:23:21 +04:00
2016-01-04 04:20:54 +03:00
self . retry = builder ( ' retry ' , retry_task , cp = False )
2010-09-26 04:35:55 +04:00
self . need_retry = False
2010-09-25 23:51:51 +04:00
def kill_kids ( self ) :
2010-09-26 04:35:55 +04:00
if self . tail_proc is not None :
self . tail_proc . terminate ( )
self . tail_proc . wait ( )
self . tail_proc = None
if self . retry is not None :
self . retry . proc . terminate ( )
self . retry . proc . wait ( )
self . retry = None
2010-09-25 23:51:51 +04:00
for b in self . tlist :
if b . proc is not None :
2019-05-03 08:05:08 +03:00
run_cmd ( " killbysubdir %s > /dev/null 2>&1 " % b . test_source_dir , checkfail = False )
2010-09-25 23:51:51 +04:00
b . proc . terminate ( )
b . proc . wait ( )
b . proc = None
def wait_one ( self ) :
while True :
none_running = True
for b in self . tlist :
if b . proc is None :
continue
none_running = False
b . status = b . proc . poll ( )
if b . status is None :
continue
b . proc = None
return b
2010-09-26 04:35:55 +04:00
if options . retry :
ret = self . retry . proc . poll ( )
if ret is not None :
self . need_retry = True
self . retry = None
return None
2010-09-25 23:51:51 +04:00
if none_running :
return None
time . sleep ( 0.1 )
def run ( self ) :
while True :
b = self . wait_one ( )
2010-09-26 04:35:55 +04:00
if options . retry and self . need_retry :
self . kill_kids ( )
2017-01-11 17:02:17 +03:00
do_print ( " retry needed " )
2010-10-02 04:58:47 +04:00
return ( 0 , None , None , None , " retry " )
2010-09-25 23:51:51 +04:00
if b is None :
break
if os . WIFSIGNALED ( b . status ) or os . WEXITSTATUS ( b . status ) != 0 :
self . kill_kids ( )
2010-10-01 14:19:56 +04:00
return ( b . status , b . name , b . stage , b . tag , " %s : [ %s ] failed ' %s ' with status %d " % ( b . name , b . stage , b . cmd , b . status ) )
2010-09-25 23:51:51 +04:00
b . start_next ( )
self . kill_kids ( )
2010-10-01 14:19:56 +04:00
return ( 0 , None , None , None , " All OK " )
2010-09-25 23:51:51 +04:00
2019-05-21 04:56:12 +03:00
def write_system_info ( self , filename ) :
with open ( filename , ' w ' ) as f :
for cmd in [ ' uname -a ' ,
' lsb_release -a ' ,
' free ' ,
' mount ' ,
' cat /proc/cpuinfo ' ,
' cc --version ' ,
' df -m . ' ,
' df -m %s ' % testbase ] :
out = run_cmd ( cmd , output = True , checkfail = False )
print ( ' ### %s ' % cmd , file = f )
print ( out , file = f )
print ( file = f )
2015-07-01 01:45:47 +03:00
2010-09-25 23:51:51 +04:00
def tarlogs ( self , fname ) :
2019-05-21 04:56:12 +03:00
with tarfile . open ( fname , " w:gz " ) as tar :
for b in self . tlist :
tar . add ( b . stdout_path , arcname = " %s .stdout " % b . tag )
tar . add ( b . stderr_path , arcname = " %s .stderr " % b . tag )
if os . path . exists ( " autobuild.log " ) :
tar . add ( " autobuild.log " )
filename = ' system-info.txt '
self . write_system_info ( filename )
tar . add ( filename )
2010-09-25 23:51:51 +04:00
def remove_logs ( self ) :
for b in self . tlist :
os . unlink ( b . stdout_path )
os . unlink ( b . stderr_path )
def start_tail ( self ) :
2018-08-27 12:13:29 +03:00
cmd = [ " tail " , " -f " ]
for b in self . tlist :
cmd . append ( b . stdout_path )
cmd . append ( b . stderr_path )
self . tail_proc = Popen ( cmd , close_fds = True )
2010-09-25 23:51:51 +04:00
def cleanup ( ) :
2010-09-26 02:30:13 +04:00
if options . nocleanup :
return
2017-02-21 19:05:08 +03:00
run_cmd ( " stat %s || true " % test_tmpdir , show = True )
2017-01-11 16:13:00 +03:00
run_cmd ( " stat %s " % testbase , show = True )
2017-04-13 19:46:50 +03:00
do_print ( " Cleaning up %r " % cleanup_list )
2010-09-25 23:51:51 +04:00
for d in cleanup_list :
run_cmd ( " rm -rf %s " % d )
2010-10-01 04:53:38 +04:00
def daemonize ( logfile ) :
pid = os . fork ( )
2018-07-30 09:19:33 +03:00
if pid == 0 : # Parent
2010-10-01 04:53:38 +04:00
os . setsid ( )
pid = os . fork ( )
2018-07-30 09:19:33 +03:00
if pid != 0 : # Actual daemon
2010-10-01 04:53:38 +04:00
os . _exit ( 0 )
2018-07-30 09:19:33 +03:00
else : # Grandparent
2010-10-01 04:53:38 +04:00
os . _exit ( 0 )
import resource # Resource usage information.
maxfd = resource . getrlimit ( resource . RLIMIT_NOFILE ) [ 1 ]
if maxfd == resource . RLIM_INFINITY :
2018-07-30 09:19:33 +03:00
maxfd = 1024 # Rough guess at maximum number of open file descriptors.
2010-10-01 04:53:38 +04:00
for fd in range ( 0 , maxfd ) :
try :
os . close ( fd )
except OSError :
pass
os . open ( logfile , os . O_RDWR | os . O_CREAT )
os . dup2 ( 0 , 1 )
os . dup2 ( 0 , 2 )
2018-07-30 09:20:39 +03:00
2010-10-20 02:44:03 +04:00
def write_pidfile ( fname ) :
''' write a pid file, cleanup on exit '''
2019-05-21 04:56:12 +03:00
with open ( fname , mode = ' w ' ) as f :
f . write ( " %u \n " % os . getpid ( ) )
2010-10-20 02:44:03 +04:00
2010-10-01 04:53:38 +04:00
2018-07-30 09:19:21 +03:00
def rebase_tree ( rebase_url , rebase_branch = " master " ) :
2012-09-22 02:23:21 +04:00
rebase_remote = " rebaseon "
2017-01-11 17:02:17 +03:00
do_print ( " Rebasing on %s " % rebase_url )
2011-02-07 03:41:21 +03:00
run_cmd ( " git describe HEAD " , show = True , dir = test_master )
2012-09-22 02:23:21 +04:00
run_cmd ( " git remote add -t %s %s %s " %
( rebase_branch , rebase_remote , rebase_url ) ,
show = True , dir = test_master )
run_cmd ( " git fetch %s " % rebase_remote , show = True , dir = test_master )
2010-09-26 03:18:00 +04:00
if options . fix_whitespace :
2014-02-17 12:15:30 +04:00
run_cmd ( " git rebase --force-rebase --whitespace=fix %s / %s " %
2012-09-22 02:23:21 +04:00
( rebase_remote , rebase_branch ) ,
show = True , dir = test_master )
2010-09-26 03:18:00 +04:00
else :
2014-02-17 12:15:30 +04:00
run_cmd ( " git rebase --force-rebase %s / %s " %
2012-09-22 02:23:21 +04:00
( rebase_remote , rebase_branch ) ,
show = True , dir = test_master )
diff = run_cmd ( " git --no-pager diff HEAD %s / %s " %
( rebase_remote , rebase_branch ) ,
dir = test_master , output = True )
2010-09-26 22:46:01 +04:00
if diff == ' ' :
2017-01-11 17:02:17 +03:00
do_print ( " No differences between HEAD and %s / %s - exiting " %
2018-07-30 09:16:12 +03:00
( rebase_remote , rebase_branch ) )
2010-09-26 22:46:01 +04:00
sys . exit ( 0 )
2012-09-22 02:23:21 +04:00
run_cmd ( " git describe %s / %s " %
( rebase_remote , rebase_branch ) ,
show = True , dir = test_master )
2011-02-07 03:41:21 +03:00
run_cmd ( " git describe HEAD " , show = True , dir = test_master )
2012-09-22 02:23:21 +04:00
run_cmd ( " git --no-pager diff --stat HEAD %s / %s " %
( rebase_remote , rebase_branch ) ,
show = True , dir = test_master )
2010-09-26 01:09:11 +04:00
2018-07-30 09:20:39 +03:00
2018-07-30 09:19:21 +03:00
def push_to ( push_url , push_branch = " master " ) :
2012-09-22 04:18:11 +04:00
push_remote = " pushto "
2017-01-11 17:02:17 +03:00
do_print ( " Pushing to %s " % push_url )
2010-09-26 02:30:13 +04:00
if options . mark :
2010-10-03 18:53:45 +04:00
run_cmd ( " git config --replace-all core.editor script/commit_mark.sh " , dir = test_master )
run_cmd ( " git commit --amend -c HEAD " , dir = test_master )
2010-10-01 01:42:02 +04:00
# the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
# run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
2012-09-22 04:18:11 +04:00
run_cmd ( " git remote add -t %s %s %s " %
( push_branch , push_remote , push_url ) ,
show = True , dir = test_master )
run_cmd ( " git push %s +HEAD: %s " %
( push_remote , push_branch ) ,
show = True , dir = test_master )
2010-09-26 02:30:13 +04:00
2018-07-30 09:21:29 +03:00
2015-04-14 08:18:02 +03:00
def send_email ( subject , text , log_tar ) :
2018-03-23 01:24:16 +03:00
if options . email is None :
do_print ( " not sending email because the recipient is not set " )
2018-09-03 13:20:31 +03:00
do_print ( " the text content would have been: \n \n Subject: %s \n \n %s " %
2018-03-23 01:24:16 +03:00
( subject , text ) )
return
2015-04-14 08:18:02 +03:00
outer = MIMEMultipart ( )
outer [ ' Subject ' ] = subject
outer [ ' To ' ] = options . email
outer [ ' From ' ] = options . email_from
2018-07-30 09:19:21 +03:00
outer [ ' Date ' ] = email . utils . formatdate ( localtime = True )
2015-04-14 08:18:02 +03:00
outer . preamble = ' Autobuild mails are now in MIME because we optionally attach the logs. \n '
outer . attach ( MIMEText ( text , ' plain ' ) )
if options . attach_logs :
2019-05-21 04:56:12 +03:00
with open ( log_tar , ' rb ' ) as fp :
msg = MIMEApplication ( fp . read ( ) , ' gzip ' , email . encoders . encode_base64 )
2015-04-14 08:18:02 +03:00
# Set the filename parameter
msg . add_header ( ' Content-Disposition ' , ' attachment ' , filename = os . path . basename ( log_tar ) )
outer . attach ( msg )
content = outer . as_string ( )
s = smtplib . SMTP ( options . email_server )
2019-04-03 03:17:17 +03:00
email_user = os . getenv ( ' SMTP_USERNAME ' )
email_password = os . getenv ( ' SMTP_PASSWORD ' )
if email_user is not None :
s . starttls ( )
s . login ( email_user , email_password )
2015-04-14 08:18:02 +03:00
s . sendmail ( options . email_from , [ options . email ] , content )
s . set_debuglevel ( 1 )
s . quit ( )
2010-09-30 21:41:36 +04:00
2018-07-30 09:20:39 +03:00
2015-05-28 04:46:23 +03:00
def email_failure ( status , failed_task , failed_stage , failed_tag , errstr ,
2016-03-14 04:18:54 +03:00
elapsed_time , log_base = None , add_log_tail = True ) :
2010-09-30 21:41:36 +04:00
''' send an email to options.email about the failure '''
2015-05-28 04:46:23 +03:00
elapsed_minutes = elapsed_time / 60.0
2012-09-22 04:34:38 +04:00
if log_base is None :
2012-09-24 13:53:22 +04:00
log_base = gitroot
2010-09-30 21:41:36 +04:00
text = '''
Dear Developer ,
2015-05-28 04:46:23 +03:00
Your autobuild on % s failed after % .1 f minutes
when trying to test % s with the following error :
2010-09-30 21:41:36 +04:00
% s
the autobuild has been abandoned . Please fix the error and resubmit .
2010-10-12 06:06:43 +04:00
A summary of the autobuild process is here :
2012-09-22 04:34:38 +04:00
% s / autobuild . log
2015-05-28 04:46:23 +03:00
''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
2015-05-04 10:21:53 +03:00
2016-07-27 05:28:04 +03:00
if options . restrict_tests :
text + = """
The build was restricted to tests matching % s \n """ % o ptions.restrict_tests
2010-10-12 06:06:43 +04:00
if failed_task != ' rebase ' :
text + = '''
2010-09-30 21:41:36 +04:00
You can see logs of the failed task here :
2010-09-25 23:51:51 +04:00
2012-09-22 04:34:38 +04:00
% s / % s . stdout
% s / % s . stderr
2010-09-30 21:41:36 +04:00
or you can get full logs of all tasks in this job here :
2012-09-22 04:34:38 +04:00
% s / logs . tar . gz
2010-09-30 21:41:36 +04:00
2010-10-02 06:53:34 +04:00
The top commit for the tree that was built was :
% s
2012-09-22 04:34:38 +04:00
''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
2015-04-14 08:18:02 +03:00
2016-03-14 04:18:54 +03:00
if add_log_tail :
f = open ( " %s / %s .stdout " % ( gitroot , failed_tag ) , ' r ' )
lines = f . readlines ( )
log_tail = " " . join ( lines [ - 50 : ] )
num_lines = len ( lines )
if num_lines < 50 :
# Also include stderr (compile failures) if < 50 lines of stdout
f = open ( " %s / %s .stderr " % ( gitroot , failed_tag ) , ' r ' )
2018-07-30 09:18:25 +03:00
log_tail + = " " . join ( f . readlines ( ) [ - ( 50 - num_lines ) : ] )
2016-03-14 04:18:54 +03:00
text + = '''
The last 50 lines of log messages :
% s
''' % lo g_tail
f . close ( )
2015-04-14 08:18:02 +03:00
logs = os . path . join ( gitroot , ' logs.tar.gz ' )
2016-08-10 13:44:26 +03:00
send_email ( ' autobuild[ %s ] failure on %s for task %s during %s '
% ( options . branch , platform . node ( ) , failed_task , failed_stage ) ,
2015-04-14 08:18:02 +03:00
text , logs )
2010-09-25 23:51:51 +04:00
2018-07-30 09:20:39 +03:00
2015-05-28 04:46:23 +03:00
def email_success ( elapsed_time , log_base = None ) :
2010-10-01 13:28:48 +04:00
''' send an email to options.email about a successful build '''
2012-09-22 04:34:38 +04:00
if log_base is None :
2012-09-24 13:53:22 +04:00
log_base = gitroot
2010-10-01 13:28:48 +04:00
text = '''
Dear Developer ,
2015-05-28 04:46:23 +03:00
Your autobuild on % s has succeeded after % .1 f minutes .
2010-10-01 13:28:48 +04:00
2015-05-28 04:46:23 +03:00
''' % (platform.node(), elapsed_time / 60.)
2010-10-01 13:28:48 +04:00
2016-07-27 05:28:04 +03:00
if options . restrict_tests :
text + = """
The build was restricted to tests matching % s \n """ % o ptions.restrict_tests
2010-10-01 13:28:48 +04:00
if options . keeplogs :
text + = '''
you can get full logs of all tasks in this job here :
2012-09-22 04:34:38 +04:00
% s / logs . tar . gz
2010-10-01 13:28:48 +04:00
2012-09-22 04:34:38 +04:00
''' % lo g_base
2010-10-02 06:53:34 +04:00
text + = '''
The top commit for the tree that was built was :
% s
''' % top_commit_msg
2015-04-14 08:18:02 +03:00
logs = os . path . join ( gitroot , ' logs.tar.gz ' )
2016-08-10 13:44:26 +03:00
send_email ( ' autobuild[ %s ] success on %s ' % ( options . branch , platform . node ( ) ) ,
2015-04-14 08:18:02 +03:00
text , logs )
2010-10-01 13:28:48 +04:00
2010-10-02 06:53:34 +04:00
# get the top commit message, for emails
top_commit_msg = run_cmd ( " git log -1 " , dir = gitroot , output = True )
2010-09-25 23:51:51 +04:00
try :
2010-09-26 01:09:11 +04:00
os . makedirs ( testbase )
2018-02-14 00:35:53 +03:00
except Exception as reason :
2010-09-26 01:09:11 +04:00
raise Exception ( " Unable to create %s : %s " % ( testbase , reason ) )
cleanup_list . append ( testbase )
2010-09-25 23:51:51 +04:00
2010-10-01 04:53:38 +04:00
if options . daemon :
logfile = os . path . join ( testbase , " log " )
2017-01-11 17:02:17 +03:00
do_print ( " Forking into the background, writing progress to %s " % logfile )
2010-10-01 04:53:38 +04:00
daemonize ( logfile )
2010-10-20 02:44:03 +04:00
write_pidfile ( gitroot + " /autobuild.pid " )
2015-05-28 04:46:23 +03:00
start_time = time . time ( )
2010-09-26 04:35:55 +04:00
while True :
try :
2017-01-11 16:13:00 +03:00
run_cmd ( " rm -rf %s " % test_tmpdir , show = True )
2016-12-16 15:35:36 +03:00
os . makedirs ( test_tmpdir )
2017-03-20 06:49:34 +03:00
# The waf uninstall code removes empty directories all the way
# up the tree. Creating a file in test_tmpdir stops it from
# being removed.
run_cmd ( " touch %s " % os . path . join ( test_tmpdir ,
" .directory-is-not-empty " ) , show = True )
2017-01-11 16:13:00 +03:00
run_cmd ( " stat %s " % test_tmpdir , show = True )
run_cmd ( " stat %s " % testbase , show = True )
2015-03-28 19:15:03 +03:00
run_cmd ( " git clone --recursive --shared %s %s " % ( gitroot , test_master ) , show = True , dir = gitroot )
2012-01-24 04:43:46 +04:00
except Exception :
2010-09-26 04:35:55 +04:00
cleanup ( )
raise
try :
2019-05-23 05:07:49 +03:00
if options . rebase is not None :
rebase_tree ( options . rebase , rebase_branch = options . branch )
except Exception :
cleanup_list . append ( gitroot + " /autobuild.pid " )
cleanup ( )
elapsed_time = time . time ( ) - start_time
email_failure ( - 1 , ' rebase ' , ' rebase ' , ' rebase ' ,
' rebase on %s failed ' % options . branch ,
elapsed_time , log_base = options . log_base )
sys . exit ( 1 )
try :
2016-07-27 05:09:08 +03:00
blist = buildlist ( args , options . rebase , rebase_branch = options . branch )
2010-09-26 04:35:55 +04:00
if options . tail :
blist . start_tail ( )
2010-10-01 14:19:56 +04:00
( status , failed_task , failed_stage , failed_tag , errstr ) = blist . run ( )
2010-09-26 04:35:55 +04:00
if status != 0 or errstr != " retry " :
break
cleanup ( )
2012-01-24 04:43:46 +04:00
except Exception :
2010-09-26 04:35:55 +04:00
cleanup ( )
raise
2010-09-25 23:51:51 +04:00
2010-10-21 13:41:06 +04:00
cleanup_list . append ( gitroot + " /autobuild.pid " )
2017-01-11 16:13:00 +03:00
do_print ( errstr )
2010-09-25 23:51:51 +04:00
blist . kill_kids ( )
if options . tail :
2017-01-11 17:02:17 +03:00
do_print ( " waiting for tail to flush " )
2010-09-25 23:51:51 +04:00
time . sleep ( 1 )
2015-05-28 04:46:23 +03:00
elapsed_time = time . time ( ) - start_time
2010-09-25 23:51:51 +04:00
if status == 0 :
2010-09-26 02:30:13 +04:00
if options . passcmd is not None :
2017-01-11 17:02:17 +03:00
do_print ( " Running passcmd: %s " % options . passcmd )
2010-09-26 02:30:13 +04:00
run_cmd ( options . passcmd , dir = test_master )
if options . pushto is not None :
2012-09-22 04:26:10 +04:00
push_to ( options . pushto , push_branch = options . branch )
2015-04-14 08:18:02 +03:00
if options . keeplogs or options . attach_logs :
2010-09-25 23:51:51 +04:00
blist . tarlogs ( " logs.tar.gz " )
2017-01-11 17:02:17 +03:00
do_print ( " Logs in logs.tar.gz " )
2010-10-01 13:28:48 +04:00
if options . always_email :
2015-05-28 04:46:23 +03:00
email_success ( elapsed_time , log_base = options . log_base )
2010-09-25 23:51:51 +04:00
blist . remove_logs ( )
cleanup ( )
2017-01-11 17:02:17 +03:00
do_print ( errstr )
2010-09-25 23:51:51 +04:00
sys . exit ( 0 )
# something failed, gather a tar of the logs
blist . tarlogs ( " logs.tar.gz " )
2010-09-30 21:41:36 +04:00
if options . email is not None :
2015-05-28 04:46:23 +03:00
email_failure ( status , failed_task , failed_stage , failed_tag , errstr ,
elapsed_time , log_base = options . log_base )
2015-10-21 04:35:33 +03:00
else :
elapsed_minutes = elapsed_time / 60.0
2018-03-09 17:06:21 +03:00
print ( '''
2015-10-21 04:35:33 +03:00
####################################################################
AUTOBUILD FAILURE
2016-08-10 13:44:26 +03:00
Your autobuild [ % s ] on % s failed after % .1 f minutes
2015-10-21 04:35:33 +03:00
when trying to test % s with the following error :
% s
the autobuild has been abandoned . Please fix the error and resubmit .
####################################################################
2018-03-09 17:06:21 +03:00
''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
2010-09-30 21:41:36 +04:00
2010-09-25 23:51:51 +04:00
cleanup ( )
2017-01-11 17:02:17 +03:00
do_print ( errstr )
do_print ( " Logs in logs.tar.gz " )
2010-09-30 20:37:42 +04:00
sys . exit ( status )