sin/samba-mirror
1
0
Fork 0
mirror of https://github.com/samba-team/samba.git synced 2025-03-11 16:58:40 +03:00
Code

script/autobuild.py: add support for dependencies

Browse Source 
This will make it possible to split build and test stages
in the next steps.

Signed-off-by: Stefan Metzmacher <metze@samba.org>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Reviewed-by: Andreas Schneider <asn@samba.org>
This commit is contained in:
Stefan Metzmacher 2020-12-17 11:53:21 +01:00
parent 404cd173e9
commit 95849d3d6a
1 changed files with 94 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
script/autobuild.py
View File

@ -54,6 +54,7 @@ parser = OptionParser()
parser.add_option("--tail", help="show output while running", default=False, action="store_true")
parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
default=def_testbase)
parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
@ -160,7 +161,7 @@ def format_option(name, value=None):
def make_test(
cmd='make test',
cmd='make testonly',
FAIL_IMMEDIATELY=1,
INJECT_SELFTEST_PREFIX=1,
TESTS='',
@ -954,6 +955,8 @@ class builder(object):
self.git_clone_required = False
if "git-clone-required" in definition:
self.git_clone_required = bool(definition["git-clone-required"])
self.proc = None
self.done = False
self.next = 0
self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
@ -969,22 +972,57 @@ class builder(object):
self.cwd = "%s/%s" % (self.builder_dir, self.dir)
self.selftest_prefix = "%s/bin/ab" % (self.cwd)
self.prefix = "%s/%s" % (test_prefix, self.tag)
self.consumers = []
self.producer = None
if self.git_clone_required:
assert "dependency" not in definition
def mark_existing(self):
do_print('%s: Mark as existing dependency' % self.name)
self.next = len(self.sequence)
self.done = True
def add_consumer(self, consumer):
do_print("%s: add consumer: %s" % (self.name, consumer.name))
consumer.producer = self
consumer.test_source_dir = self.test_source_dir
self.consumers.append(consumer)
def start_next(self):
if self.producer is not None:
if not self.producer.done:
do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
return
if self.next == 0:
rmdir_force(self.builder_dir)
rmdir_force(self.prefix)
if not self.git_clone_required:
if self.producer is not None:
run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
elif not self.git_clone_required:
run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
else:
run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
if self.next == len(self.sequence):
if not options.nocleanup:
if not self.done:
do_print('%s: Completed OK' % self.name)
self.done = True
if not options.nocleanup and len(self.consumers) == 0:
do_print('%s: Cleaning up' % self.name)
rmdir_force(self.builder_dir)
rmdir_force(self.prefix)
do_print('%s: Completed OK' % self.name)
self.done = True
for consumer in self.consumers:
if consumer.next != 0:
continue
do_print('%s: Starting consumer %s' % (self.name, consumer.name))
consumer.start_next()
if self.producer is not None:
self.producer.consumers.remove(self)
assert self.producer.done
self.producer.start_next()
do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
return
(self.stage, self.cmd) = self.sequence[self.next]
self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
@ -1002,6 +1040,18 @@ class builder(object):
stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
self.next += 1
def expand_dependencies(n):
deps = list()
if "dependency" in tasks[n]:
depname = tasks[n]["dependency"]
assert depname in tasks
sdeps = expand_dependencies(depname)
assert n not in sdeps
for sdep in sdeps:
deps.append(sdep)
deps.append(depname)
return deps
class buildlist(object):
'''handle build of multiple directories'''
@ -1015,6 +1065,22 @@ class buildlist(object):
else:
tasknames = defaulttasks
given_tasknames = tasknames.copy()
implicit_tasknames = []
for n in given_tasknames:
deps = expand_dependencies(n)
for dep in deps:
if dep in given_tasknames:
continue
if dep in implicit_tasknames:
continue
implicit_tasknames.append(dep)
tasknames = implicit_tasknames.copy()
tasknames.extend(given_tasknames)
do_print("given_tasknames: %s" % given_tasknames)
do_print("implicit_tasknames: %s" % implicit_tasknames)
do_print("tasknames: %s" % tasknames)
self.tlist = [builder(n, tasks[n]) for n in tasknames]
if options.retry:
@ -1044,6 +1110,21 @@ class buildlist(object):
self.retry = builder('retry', retry_task)
self.need_retry = False
if options.skip_dependencies:
for b in self.tlist:
if b.name in implicit_tasknames:
b.mark_existing()
for b in self.tlist:
do_print("b.name=%s" % b.name)
if "dependency" not in b.definition:
continue
depname = b.definition["dependency"]
do_print("b.name=%s: dependency:%s" % (b.name, depname))
for p in self.tlist:
if p.name == depname:
p.add_consumer(b)
def kill_kids(self):
if self.tail_proc is not None:
self.tail_proc.terminate()
@ -1377,7 +1458,10 @@ The top commit for the tree that was built was:
top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
try:
os.makedirs(testbase)
if options.skip_dependencies:
run_cmd("stat %s" % testbase, dir=testbase, output=True)
else:
os.makedirs(testbase)
except Exception as reason:
raise Exception("Unable to create %s : %s" % (testbase, reason))
cleanup_list.append(testbase)
@ -1402,7 +1486,10 @@ while True:
".directory-is-not-empty"), show=True)
run_cmd("stat %s" % test_tmpdir, show=True)
run_cmd("stat %s" % testbase, show=True)
run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
if options.skip_dependencies:
run_cmd("stat %s" % test_master, dir=testbase, output=True)
else:
run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
except Exception:
cleanup()
raise