2010-09-25 23:51:51 +04:00
#!/usr/bin/env python
# run tests on all Samba subprojects and push to a git tree on success
# Copyright Andrew Tridgell 2010
# released under GNU GPL v3 or later
from subprocess import Popen , PIPE
import os , signal , tarfile , sys , time
from optparse import OptionParser
2010-09-26 02:53:32 +04:00
samba_master = os . getenv ( ' SAMBA_MASTER ' , ' git://git.samba.org/samba.git ' )
samba_master_ssh = os . getenv ( ' SAMBA_MASTER_SSH ' , ' git+ssh://git.samba.org/data/git/samba.git ' )
2010-09-25 23:51:51 +04:00
cleanup_list = [ ]
2010-09-26 03:32:55 +04:00
os . putenv ( ' CC ' , " ccache gcc " )
2010-09-25 23:51:51 +04:00
tasks = {
" source3 " : [ " ./autogen.sh " ,
" ./configure.developer $ {PREFIX} " ,
" make basics " ,
2010-09-26 02:53:32 +04:00
" make -j 4 everything " , # don't use too many processes
" make install " ,
2010-09-28 22:23:35 +04:00
" TDB_NO_FSYNC=1 make test FAIL_IMMEDIATELY=1 " ] ,
2010-09-25 23:51:51 +04:00
" source4 " : [ " ./autogen.sh " ,
" ./configure.developer $ {PREFIX} " ,
" make -j " ,
2010-09-26 02:53:32 +04:00
" make install " ,
2010-09-28 22:23:35 +04:00
" TDB_NO_FSYNC=1 make test FAIL_IMMEDIATELY=1 " ] ,
2010-09-25 23:51:51 +04:00
" source4/lib/ldb " : [ " ./autogen-waf.sh " ,
" ./configure --enable-developer -C $ {PREFIX} " ,
" make -j " ,
2010-09-26 02:53:32 +04:00
" make install " ,
2010-09-25 23:51:51 +04:00
" make test " ] ,
" lib/tdb " : [ " ./autogen-waf.sh " ,
" ./configure --enable-developer -C $ {PREFIX} " ,
" make -j " ,
2010-09-26 02:53:32 +04:00
" make install " ,
2010-09-25 23:51:51 +04:00
" make test " ] ,
" lib/talloc " : [ " ./autogen-waf.sh " ,
" ./configure --enable-developer -C $ {PREFIX} " ,
" make -j " ,
2010-09-26 02:53:32 +04:00
" make install " ,
2010-09-25 23:51:51 +04:00
" make test " ] ,
" lib/replace " : [ " ./autogen-waf.sh " ,
" ./configure --enable-developer -C $ {PREFIX} " ,
" make -j " ,
2010-09-26 02:53:32 +04:00
" make install " ,
2010-09-25 23:51:51 +04:00
" make test " ] ,
" lib/tevent " : [ " ./autogen-waf.sh " ,
" ./configure --enable-developer -C $ {PREFIX} " ,
" make -j " ,
2010-09-26 02:53:32 +04:00
" make install " ,
2010-09-25 23:51:51 +04:00
" make test " ] ,
}
2010-09-26 04:35:55 +04:00
retry_task = [ ''' set -e
git remote add - t master master % s
2010-09-26 06:43:13 +04:00
git fetch master
2010-09-26 04:35:55 +04:00
while : ; do
sleep 60
2010-09-26 06:43:13 +04:00
git describe master / master > old_master . desc
2010-09-26 04:35:55 +04:00
git fetch master
2010-09-26 06:43:13 +04:00
git describe master / master > master . desc
diff old_master . desc master . desc
2010-09-26 04:35:55 +04:00
done
''' % s amba_master]
2010-09-26 23:50:06 +04:00
def run_cmd ( cmd , dir = " . " , show = None , output = False , checkfail = True ) :
2010-09-25 23:51:51 +04:00
cwd = os . getcwd ( )
os . chdir ( dir )
if show is None :
show = options . verbose
if show :
print ( " Running: ' %s ' in ' %s ' " % ( cmd , dir ) )
2010-09-26 22:46:01 +04:00
if output :
ret = Popen ( [ cmd ] , shell = True , stdout = PIPE ) . communicate ( ) [ 0 ]
os . chdir ( cwd )
return ret
2010-09-25 23:51:51 +04:00
ret = os . system ( cmd )
os . chdir ( cwd )
2010-09-26 23:50:06 +04:00
if checkfail and ret != 0 :
2010-09-25 23:51:51 +04:00
raise Exception ( " FAILED %s : %d " % ( cmd , ret ) )
2010-09-26 23:50:06 +04:00
return ret
2010-09-25 23:51:51 +04:00
class builder :
''' handle build of one directory '''
def __init__ ( self , name , sequence ) :
self . name = name
2010-09-26 04:35:55 +04:00
if name in [ ' pass ' , ' fail ' , ' retry ' ] :
2010-09-26 02:30:13 +04:00
self . dir = " . "
2010-09-26 04:35:55 +04:00
else :
self . dir = self . name
2010-09-25 23:51:51 +04:00
self . tag = self . name . replace ( ' / ' , ' _ ' )
self . sequence = sequence
self . next = 0
2010-09-26 01:09:11 +04:00
self . stdout_path = " %s / %s .stdout " % ( testbase , self . tag )
self . stderr_path = " %s / %s .stderr " % ( testbase , self . tag )
2010-09-25 23:51:51 +04:00
cleanup_list . append ( self . stdout_path )
cleanup_list . append ( self . stderr_path )
run_cmd ( " rm -f %s %s " % ( self . stdout_path , self . stderr_path ) )
self . stdout = open ( self . stdout_path , ' w ' )
self . stderr = open ( self . stderr_path , ' w ' )
self . stdin = open ( " /dev/null " , ' r ' )
2010-09-26 01:09:11 +04:00
self . sdir = " %s / %s " % ( testbase , self . tag )
self . prefix = " %s /prefix/ %s " % ( testbase , self . tag )
2010-09-25 23:51:51 +04:00
run_cmd ( " rm -rf %s " % self . sdir )
cleanup_list . append ( self . sdir )
cleanup_list . append ( self . prefix )
2010-09-26 01:09:11 +04:00
os . makedirs ( self . sdir )
2010-09-25 23:51:51 +04:00
run_cmd ( " rm -rf %s " % self . sdir )
run_cmd ( " git clone --shared %s %s " % ( gitroot , self . sdir ) )
self . start_next ( )
def start_next ( self ) :
if self . next == len ( self . sequence ) :
print ' %s : Completed OK ' % self . name
self . done = True
return
self . cmd = self . sequence [ self . next ] . replace ( " $ {PREFIX} " , " --prefix= %s " % self . prefix )
print ' %s : Running %s ' % ( self . name , self . cmd )
cwd = os . getcwd ( )
2010-09-26 02:30:13 +04:00
os . chdir ( " %s / %s " % ( self . sdir , self . dir ) )
2010-09-25 23:51:51 +04:00
self . proc = Popen ( self . cmd , shell = True ,
stdout = self . stdout , stderr = self . stderr , stdin = self . stdin )
os . chdir ( cwd )
self . next + = 1
class buildlist :
''' handle build of multiple directories '''
def __init__ ( self , tasklist , tasknames ) :
2010-09-26 03:18:00 +04:00
global tasks
2010-09-25 23:51:51 +04:00
self . tlist = [ ]
self . tail_proc = None
2010-09-26 04:35:55 +04:00
self . retry = None
2010-09-26 02:30:13 +04:00
if tasknames == [ ' pass ' ] :
tasks = { ' pass ' : [ ' /bin/true ' ] }
if tasknames == [ ' fail ' ] :
tasks = { ' fail ' : [ ' /bin/false ' ] }
2010-09-25 23:51:51 +04:00
if tasknames == [ ] :
tasknames = tasklist
for n in tasknames :
b = builder ( n , tasks [ n ] )
self . tlist . append ( b )
2010-09-26 04:35:55 +04:00
if options . retry :
self . retry = builder ( ' retry ' , retry_task )
self . need_retry = False
2010-09-25 23:51:51 +04:00
def kill_kids ( self ) :
2010-09-26 04:35:55 +04:00
if self . tail_proc is not None :
self . tail_proc . terminate ( )
self . tail_proc . wait ( )
self . tail_proc = None
if self . retry is not None :
self . retry . proc . terminate ( )
self . retry . proc . wait ( )
self . retry = None
2010-09-25 23:51:51 +04:00
for b in self . tlist :
if b . proc is not None :
2010-09-26 23:50:06 +04:00
run_cmd ( " killbysubdir %s > /dev/null 2>&1 " % b . sdir , checkfail = False )
2010-09-25 23:51:51 +04:00
b . proc . terminate ( )
b . proc . wait ( )
b . proc = None
def wait_one ( self ) :
while True :
none_running = True
for b in self . tlist :
if b . proc is None :
continue
none_running = False
b . status = b . proc . poll ( )
if b . status is None :
continue
b . proc = None
return b
2010-09-26 04:35:55 +04:00
if options . retry :
ret = self . retry . proc . poll ( )
if ret is not None :
self . need_retry = True
self . retry = None
return None
2010-09-25 23:51:51 +04:00
if none_running :
return None
time . sleep ( 0.1 )
def run ( self ) :
while True :
b = self . wait_one ( )
2010-09-26 04:35:55 +04:00
if options . retry and self . need_retry :
self . kill_kids ( )
print ( " retry needed " )
return ( 0 , " retry " )
2010-09-25 23:51:51 +04:00
if b is None :
break
if os . WIFSIGNALED ( b . status ) or os . WEXITSTATUS ( b . status ) != 0 :
self . kill_kids ( )
return ( b . status , " %s : failed ' %s ' with status %d " % ( b . name , b . cmd , b . status ) )
b . start_next ( )
self . kill_kids ( )
return ( 0 , " All OK " )
def tarlogs ( self , fname ) :
tar = tarfile . open ( fname , " w:gz " )
for b in self . tlist :
tar . add ( b . stdout_path , arcname = " %s .stdout " % b . tag )
tar . add ( b . stderr_path , arcname = " %s .stderr " % b . tag )
tar . close ( )
def remove_logs ( self ) :
for b in self . tlist :
os . unlink ( b . stdout_path )
os . unlink ( b . stderr_path )
def start_tail ( self ) :
cwd = os . getcwd ( )
2010-09-26 01:09:11 +04:00
cmd = " tail -f *.stdout *.stderr "
os . chdir ( testbase )
2010-09-25 23:51:51 +04:00
self . tail_proc = Popen ( cmd , shell = True )
os . chdir ( cwd )
def cleanup ( ) :
2010-09-26 02:30:13 +04:00
if options . nocleanup :
return
2010-09-25 23:51:51 +04:00
print ( " Cleaning up .... " )
for d in cleanup_list :
run_cmd ( " rm -rf %s " % d )
def find_git_root ( ) :
''' get to the top of the git repo '''
cwd = os . getcwd ( )
while os . getcwd ( ) != ' / ' :
try :
os . stat ( " .git " )
ret = os . getcwd ( )
os . chdir ( cwd )
return ret
except :
os . chdir ( " .. " )
pass
os . chdir ( cwd )
return None
2010-09-26 01:09:11 +04:00
def rebase_tree ( url ) :
print ( " Rebasing on %s " % url )
run_cmd ( " git remote add -t master master %s " % url , show = True , dir = test_master )
run_cmd ( " git fetch master " , show = True , dir = test_master )
2010-09-26 03:18:00 +04:00
if options . fix_whitespace :
run_cmd ( " git rebase --whitespace=fix master/master " , show = True , dir = test_master )
else :
run_cmd ( " git rebase master/master " , show = True , dir = test_master )
2010-09-26 22:46:01 +04:00
diff = run_cmd ( " git --no-pager diff HEAD master/master " , dir = test_master , output = True )
if diff == ' ' :
print ( " No differences between HEAD and master/master - exiting " )
sys . exit ( 0 )
2010-09-26 01:09:11 +04:00
2010-09-26 02:30:13 +04:00
def push_to ( url ) :
print ( " Pushing to %s " % url )
if options . mark :
2010-09-28 22:24:37 +04:00
run_cmd ( " EDITOR=script/commit_mark.sh git notes edit HEAD " , dir = test_master )
2010-09-26 02:30:13 +04:00
run_cmd ( " git remote add -t master pushto %s " % url , show = True , dir = test_master )
run_cmd ( " git push pushto +HEAD:master " , show = True , dir = test_master )
2010-09-26 01:09:11 +04:00
def_testbase = os . getenv ( " AUTOBUILD_TESTBASE " , " /memdisk/ %s " % os . getenv ( ' USER ' ) )
2010-09-25 23:51:51 +04:00
parser = OptionParser ( )
parser . add_option ( " " , " --tail " , help = " show output while running " , default = False , action = " store_true " )
parser . add_option ( " " , " --keeplogs " , help = " keep logs " , default = False , action = " store_true " )
2010-09-26 02:30:13 +04:00
parser . add_option ( " " , " --nocleanup " , help = " don ' t remove test tree " , default = False , action = " store_true " )
2010-09-25 23:51:51 +04:00
parser . add_option ( " " , " --testbase " , help = " base directory to run tests in (default %s ) " % def_testbase ,
default = def_testbase )
2010-09-26 02:30:13 +04:00
parser . add_option ( " " , " --passcmd " , help = " command to run on success " , default = None )
2010-09-25 23:51:51 +04:00
parser . add_option ( " " , " --verbose " , help = " show all commands as they are run " ,
default = False , action = " store_true " )
2010-09-26 01:09:11 +04:00
parser . add_option ( " " , " --rebase " , help = " rebase on the given tree before testing " ,
default = None , type = ' str ' )
2010-09-26 02:53:32 +04:00
parser . add_option ( " " , " --rebase-master " , help = " rebase on %s before testing " % samba_master ,
default = False , action = ' store_true ' )
2010-09-26 02:30:13 +04:00
parser . add_option ( " " , " --pushto " , help = " push to a git url on success " ,
default = None , type = ' str ' )
2010-09-26 02:53:32 +04:00
parser . add_option ( " " , " --push-master " , help = " push to %s on success " % samba_master_ssh ,
default = False , action = ' store_true ' )
2010-09-26 02:30:13 +04:00
parser . add_option ( " " , " --mark " , help = " add a Tested-By signoff before pushing " ,
default = False , action = " store_true " )
2010-09-26 03:18:00 +04:00
parser . add_option ( " " , " --fix-whitespace " , help = " fix whitespace on rebase " ,
default = False , action = " store_true " )
2010-09-26 04:35:55 +04:00
parser . add_option ( " " , " --retry " , help = " automatically retry if master changes " ,
default = False , action = " store_true " )
2010-09-25 23:51:51 +04:00
( options , args ) = parser . parse_args ( )
2010-09-26 04:35:55 +04:00
if options . retry :
if not options . rebase_master and options . rebase is None :
raise Exception ( ' You can only use --retry if you also rebase ' )
2010-09-26 01:09:11 +04:00
testbase = " %s /build. %u " % ( options . testbase , os . getpid ( ) )
test_master = " %s /master " % testbase
2010-09-25 23:51:51 +04:00
gitroot = find_git_root ( )
if gitroot is None :
raise Exception ( " Failed to find git root " )
try :
2010-09-26 01:09:11 +04:00
os . makedirs ( testbase )
except Exception , reason :
raise Exception ( " Unable to create %s : %s " % ( testbase , reason ) )
cleanup_list . append ( testbase )
2010-09-25 23:51:51 +04:00
2010-09-26 04:35:55 +04:00
while True :
try :
run_cmd ( " rm -rf %s " % test_master )
cleanup_list . append ( test_master )
run_cmd ( " git clone --shared %s %s " % ( gitroot , test_master ) )
except :
cleanup ( )
raise
try :
if options . rebase is not None :
rebase_tree ( options . rebase )
elif options . rebase_master :
rebase_tree ( samba_master )
blist = buildlist ( tasks , args )
if options . tail :
blist . start_tail ( )
( status , errstr ) = blist . run ( )
if status != 0 or errstr != " retry " :
break
cleanup ( )
except :
cleanup ( )
raise
2010-09-25 23:51:51 +04:00
blist . kill_kids ( )
if options . tail :
print ( " waiting for tail to flush " )
time . sleep ( 1 )
if status == 0 :
print errstr
2010-09-26 02:30:13 +04:00
if options . passcmd is not None :
print ( " Running passcmd: %s " % options . passcmd )
run_cmd ( options . passcmd , dir = test_master )
if options . pushto is not None :
push_to ( options . pushto )
2010-09-26 02:53:32 +04:00
elif options . push_master :
push_to ( samba_master_ssh )
2010-09-25 23:51:51 +04:00
if options . keeplogs :
blist . tarlogs ( " logs.tar.gz " )
print ( " Logs in logs.tar.gz " )
blist . remove_logs ( )
cleanup ( )
2010-09-26 01:09:11 +04:00
print ( errstr )
2010-09-25 23:51:51 +04:00
sys . exit ( 0 )
# something failed, gather a tar of the logs
blist . tarlogs ( " logs.tar.gz " )
blist . remove_logs ( )
cleanup ( )
2010-09-26 01:09:11 +04:00
print ( errstr )
2010-09-25 23:51:51 +04:00
print ( " Logs in logs.tar.gz " )
sys . exit ( os . WEXITSTATUS ( status ) )